AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-14 10:38:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added not running module info + swapped order of module launch

Browse source
This commit is contained in:
Mokaddem 2016-08-26 09:21:02 +02:00
parent 07d7559060
commit 8b33b4e7ba
4 changed files with 93 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/LAUNCH.sh
View file

@ -112,6 +112,8 @@ function launching_scripts {
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
sleep 0.1
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
@ -159,8 +161,6 @@ function launching_scripts {
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x'
} }

154
bin/ModuleInformation.py
View file

@ -87,82 +87,100 @@ if __name__ == "__main__":
if args.clear == 1: if args.clear == 1:
clearRedisModuleInfo() clearRedisModuleInfo()
while True:
curr_range = 50 module_file_array = set()
printarray1 = [] with open('../doc/all_modules.txt', 'r') as module_file:
printarray2 = [] for line in module_file:
for queue, card in server.hgetall("queues").iteritems(): module_file_array.add(line[:-1])
key = "MODULE_" + queue + "_"
for i in range(1, 50):
curr_num = server.get("MODULE_"+ queue + "_" + str(i))
if curr_num is None:
curr_range = i
break
for moduleNum in range(1, curr_range): while True:
value = server.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")
if timestamp is not None and path is not None:
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
if int(card) > 0: all_queue = set()
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module: curr_range = 50
log = open(log_filename, 'a') printarray1 = []
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") printarray2 = []
if args.autokill == 1: printarray3 = []
kill_module(queue) for queue, card in server.hgetall("queues").iteritems():
all_queue.add(queue)
key = "MODULE_" + queue + "_"
for i in range(1, 50):
curr_num = server.get("MODULE_"+ queue + "_" + str(i))
if curr_num is None:
curr_range = i
break
printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) for moduleNum in range(1, curr_range):
value = server.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")
if timestamp is not None and path is not None:
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
else: if int(card) > 0:
printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
log = open(log_filename, 'a')
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
if args.autokill == 1:
kill_module(queue)
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
os.system('clear') else:
t1 = AsciiTable(printarray1, title="Working queues") printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
t1.column_max_width(1)
if not t1.ok:
longest_col = t1.column_widths.index(max(t1.column_widths))
max_length_col = t1.column_max_width(longest_col)
if max_length_col > 0:
for i, content in enumerate(t1.table_data):
if len(content[longest_col]) > max_length_col:
temp = ''
for l in content[longest_col].splitlines():
if len(l) > max_length_col:
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
else:
temp += l + '\n'
content[longest_col] = temp.strip()
t1.table_data[i] = content
t2 = AsciiTable(printarray2, title="Idling queues") for curr_queue in module_file_array:
t2.column_max_width(1) if curr_queue not in all_queue:
if not t2.ok: printarray3.append([curr_queue, "Not running"])
longest_col = t2.column_widths.index(max(t2.column_widths))
max_length_col = t2.column_max_width(longest_col)
if max_length_col > 0:
for i, content in enumerate(t2.table_data):
if len(content[longest_col]) > max_length_col:
temp = ''
for l in content[longest_col].splitlines():
if len(l) > max_length_col:
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
else:
temp += l + '\n'
content[longest_col] = temp.strip()
t2.table_data[i] = content
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
printarray3.insert(0,["Queue", "State"])
print t1.table os.system('clear')
print '\n' t1 = AsciiTable(printarray1, title="Working queues")
print t2.table t1.column_max_width(1)
if not t1.ok:
longest_col = t1.column_widths.index(max(t1.column_widths))
max_length_col = t1.column_max_width(longest_col)
if max_length_col > 0:
for i, content in enumerate(t1.table_data):
if len(content[longest_col]) > max_length_col:
temp = ''
for l in content[longest_col].splitlines():
if len(l) > max_length_col:
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
else:
temp += l + '\n'
content[longest_col] = temp.strip()
t1.table_data[i] = content
time.sleep(args.refresh) t2 = AsciiTable(printarray2, title="Idling queues")
t2.column_max_width(1)
if not t2.ok:
longest_col = t2.column_widths.index(max(t2.column_widths))
max_length_col = t2.column_max_width(longest_col)
if max_length_col > 0:
for i, content in enumerate(t2.table_data):
if len(content[longest_col]) > max_length_col:
temp = ''
for l in content[longest_col].splitlines():
if len(l) > max_length_col:
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
else:
temp += l + '\n'
content[longest_col] = temp.strip()
t2.table_data[i] = content
t3 = AsciiTable(printarray3, title="Not running queues")
t3.column_max_width(1)
print t1.table
print '\n'
print t2.table
print '\n'
print t3.table
time.sleep(args.refresh)

4
bin/launch_scripts.sh
View file

@ -8,6 +8,8 @@ sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
sleep 0.1
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
@ -55,5 +57,3 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x'

3
doc/generate_graph_data.py
View file

@ -32,6 +32,9 @@ with open('../bin/packages/modules.cfg', 'r') as f:
continue continue
output_set_graph = set() output_set_graph = set()
with open('all_modules.txt', 'w') as f2:
for e in all_modules:
f2.write(e+"\n")
for module in modules.keys(): for module in modules.keys():
for stream_in in modules[module]['sub']: for stream_in in modules[module]['sub']: