AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-14 02:28:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ModuleInfo support autokill and auto restart after logging.

Browse source
This commit is contained in:
Mokaddem 2016-08-24 16:52:01 +02:00
parent 3e88d0911d
commit 8fb552b7da
1 changed files with 27 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
bin/ModuleInformations.py
View file

@ -7,13 +7,33 @@ import datetime
import calendar import calendar
import redis import redis
import os import os
import signal
from subprocess import PIPE, Popen
import ConfigParser import ConfigParser
import json import json
from prettytable import PrettyTable from prettytable import PrettyTable
# CONFIG VARIABLES # CONFIG VARIABLES
threshold_stucked_module = 60*60*1 #1 hour threshold_stucked_module = 1*60*1 #1 hour
log_filename = "../logs/moduleInfo.log" log_filename = "../logs/moduleInfo.log"
command_search_pid = "ps a -o pid,cmd | grep {}"
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
def kill_module(module):
print 'trying to kill module:', module
time.sleep(8)
time.sleep(1)
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
for line in p.stdout:
splittedLine = line.split()
if 'python2' in splittedLine:
pid = int(splittedLine[0])
os.kill(pid, signal.SIGUSR1)
time.sleep(15)
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
@ -49,12 +69,14 @@ if __name__ == "__main__":
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
log = open(log_filename, 'a')
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
if int(card) > 0: if int(card) > 0:
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
log = open(log_filename, 'a')
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
kill_module(queue)
table1.add_row([num, queue, card, startTime_readable, processed_time_readable, path]) table1.add_row([num, queue, card, startTime_readable, processed_time_readable, path])
else: else:
table2.add_row([num, queue, card, startTime_readable, processed_time_readable, path]) table2.add_row([num, queue, card, startTime_readable, processed_time_readable, path])