mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 14:37:17 +00:00
ModuleInfo support autokill and auto restart after logging.
This commit is contained in:
parent
3e88d0911d
commit
8fb552b7da
1 changed files with 27 additions and 5 deletions
|
@ -7,13 +7,33 @@ import datetime
|
||||||
import calendar
|
import calendar
|
||||||
import redis
|
import redis
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
|
from subprocess import PIPE, Popen
|
||||||
import ConfigParser
|
import ConfigParser
|
||||||
import json
|
import json
|
||||||
from prettytable import PrettyTable
|
from prettytable import PrettyTable
|
||||||
|
|
||||||
# CONFIG VARIABLES
|
# CONFIG VARIABLES
|
||||||
threshold_stucked_module = 60*60*1 #1 hour
|
threshold_stucked_module = 1*60*1 #1 hour
|
||||||
log_filename = "../logs/moduleInfo.log"
|
log_filename = "../logs/moduleInfo.log"
|
||||||
|
command_search_pid = "ps a -o pid,cmd | grep {}"
|
||||||
|
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
|
||||||
|
|
||||||
|
|
||||||
|
def kill_module(module):
|
||||||
|
print 'trying to kill module:', module
|
||||||
|
time.sleep(8)
|
||||||
|
|
||||||
|
time.sleep(1)
|
||||||
|
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
||||||
|
|
||||||
|
for line in p.stdout:
|
||||||
|
splittedLine = line.split()
|
||||||
|
if 'python2' in splittedLine:
|
||||||
|
pid = int(splittedLine[0])
|
||||||
|
os.kill(pid, signal.SIGUSR1)
|
||||||
|
time.sleep(15)
|
||||||
|
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -49,12 +69,14 @@ if __name__ == "__main__":
|
||||||
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
||||||
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
||||||
|
|
||||||
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
|
|
||||||
log = open(log_filename, 'a')
|
|
||||||
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
|
||||||
|
|
||||||
if int(card) > 0:
|
if int(card) > 0:
|
||||||
|
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
|
||||||
|
log = open(log_filename, 'a')
|
||||||
|
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
||||||
|
kill_module(queue)
|
||||||
|
|
||||||
table1.add_row([num, queue, card, startTime_readable, processed_time_readable, path])
|
table1.add_row([num, queue, card, startTime_readable, processed_time_readable, path])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
table2.add_row([num, queue, card, startTime_readable, processed_time_readable, path])
|
table2.add_row([num, queue, card, startTime_readable, processed_time_readable, path])
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue