From d37b243e1dcfea8c605719eca024a4631be080cf Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 9 Dec 2016 15:13:47 +0100 Subject: [PATCH 1/2] Added legend + added more info if restarting disabled --- bin/ModuleInformation.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py index 5eb81e84..acc23a85 100755 --- a/bin/ModuleInformation.py +++ b/bin/ModuleInformation.py @@ -247,15 +247,16 @@ if __name__ == "__main__": printarray3.append([curr_queue, "No data"]) else: #If no info since long time, try to kill - if args.autokill == 1 and int(time.time()) - no_info_modules[curr_queue] > threshold_stucked_module: - kill_module(curr_queue, None) - no_info_modules[curr_queue] = int(time.time()) - printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(threshold_stucked_module - (int(time.time()) - no_info_modules[curr_queue])) + "s"]) + if args.autokill == 1: + if int(time.time()) - no_info_modules[curr_queue] > threshold_stucked_module: + kill_module(curr_queue, None) + no_info_modules[curr_queue] = int(time.time()) + printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(threshold_stucked_module - (int(time.time()) - no_info_modules[curr_queue]))) + "s"]) + else: + printarray3.append([curr_queue, "Stuck or idle, restarting disabled"]) - #printarray1.sort(lambda x,y: cmp(x[0], y[0]), reverse=False) printarray1.sort(key=lambda x: x[0][9:], reverse=False) - #printarray2.sort(lambda x,y: cmp(x[0], y[0]), reverse=False) printarray2.sort(key=lambda x: x[0][9:], reverse=False) printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) @@ -307,6 +308,12 @@ if __name__ == "__main__": t4 = AsciiTable(printarray4, title="Last actions") t4.column_max_width(1) + legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(threshold_stucked_module)]] + legend = AsciiTable(legend_array, title="Legend") + legend.column_max_width(1) + + print legend.table + print '\n' print t1.table print '\n' print t2.table From 73938cf41bd3fe15ba21045eaa76977d6837a9a5 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 22 Dec 2016 08:45:53 +0100 Subject: [PATCH 2/2] Prepared moduleInfo to handle module click with curses (will come back later). --- bin/ModuleInformation.py | 285 +++++++++++++++++++++------------------ 1 file changed, 155 insertions(+), 130 deletions(-) diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py index acc23a85..fc219815 100755 --- a/bin/ModuleInformation.py +++ b/bin/ModuleInformation.py @@ -25,9 +25,9 @@ import json from terminaltables import AsciiTable import textwrap from colorama import Fore, Back, Style, init +import curses # CONFIG VARIABLES -threshold_stucked_module = 60*10*1 #1 hour kill_retry_threshold = 60 #1m log_filename = "../logs/moduleInfo.log" command_search_pid = "ps a -o pid,cmd | grep {}" @@ -39,6 +39,15 @@ printarrayGlob = [None]*14 printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"]) lastTimeKillCommand = {} +#Curses init +#stdscr = curses.initscr() +#curses.cbreak() +#stdscr.keypad(1) + +# GLOBAL +last_refresh = 0 + + def getPid(module): p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) for line in p.stdout: @@ -145,23 +154,33 @@ def get_color(time, idle): temp = time.split(':') time = int(temp[0])*3600 + int(temp[1])*60 + int(temp[2]) - if time >= threshold_stucked_module: + if time >= args.treshold: if not idle: return Back.RED + Style.BRIGHT else: return Back.MAGENTA + Style.BRIGHT - elif time > threshold_stucked_module/2: + elif time > args.treshold/2: return Back.YELLOW + Style.BRIGHT else: return Back.GREEN + Style.BRIGHT else: return Style.RESET_ALL +def waiting_refresh(): + global last_refresh + if time.time() - last_refresh < args.refresh: + return False + else: + last_refresh = time.time() + return True + + if __name__ == "__main__": parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.') parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate') + parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate') parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)') parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)') @@ -176,8 +195,6 @@ if __name__ == "__main__": cfg = ConfigParser.ConfigParser() cfg.read(configfile) - threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module") - # REDIS # server = redis.StrictRedis( host=cfg.get("Redis_Queues", "host"), @@ -199,130 +216,138 @@ if __name__ == "__main__": cleanRedis() while True: + if waiting_refresh(): - all_queue = set() - printarray1 = [] - printarray2 = [] - printarray3 = [] - for queue, card in server.hgetall("queues").iteritems(): - all_queue.add(queue) - key = "MODULE_" + queue + "_" - keySet = "MODULE_TYPE_" + queue - array_module_type = [] - - for moduleNum in server.smembers(keySet): - value = server.get(key + str(moduleNum)) - if value is not None: - timestamp, path = value.split(", ") - if timestamp is not None and path is not None: - startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) - processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] - - if int(card) > 0: - if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module: - log = open(log_filename, 'a') - log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") - try: - last_kill_try = time.time() - lastTimeKillCommand[moduleNum] - except KeyError: - last_kill_try = kill_retry_threshold+1 - if args.autokill == 1 and last_kill_try > kill_retry_threshold : - kill_module(queue, int(moduleNum)) - - array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)]) - + #key = '' + #while key != 'q': + # key = stdsrc.getch() + # stdscr.refresh() + + all_queue = set() + printarray1 = [] + printarray2 = [] + printarray3 = [] + for queue, card in server.hgetall("queues").iteritems(): + all_queue.add(queue) + key = "MODULE_" + queue + "_" + keySet = "MODULE_TYPE_" + queue + array_module_type = [] + + for moduleNum in server.smembers(keySet): + value = server.get(key + str(moduleNum)) + if value is not None: + timestamp, path = value.split(", ") + if timestamp is not None and path is not None: + startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) + processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] + + if int(card) > 0: + if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold: + log = open(log_filename, 'a') + log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") + try: + last_kill_try = time.time() - lastTimeKillCommand[moduleNum] + except KeyError: + last_kill_try = kill_retry_threshold+1 + if args.autokill == 1 and last_kill_try > kill_retry_threshold : + kill_module(queue, int(moduleNum)) + + array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)]) + + else: + printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)]) + array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) + for e in array_module_type: + printarray1.append(e) + + for curr_queue in module_file_array: + if curr_queue not in all_queue: + printarray3.append([curr_queue, "Not running"]) + else: + if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0: + if curr_queue not in no_info_modules: + no_info_modules[curr_queue] = int(time.time()) + printarray3.append([curr_queue, "No data"]) else: - printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)]) - array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) - for e in array_module_type: - printarray1.append(e) - - for curr_queue in module_file_array: - if curr_queue not in all_queue: - printarray3.append([curr_queue, "Not running"]) - else: - if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0: - if curr_queue not in no_info_modules: - no_info_modules[curr_queue] = int(time.time()) - printarray3.append([curr_queue, "No data"]) - else: - #If no info since long time, try to kill - if args.autokill == 1: - if int(time.time()) - no_info_modules[curr_queue] > threshold_stucked_module: - kill_module(curr_queue, None) - no_info_modules[curr_queue] = int(time.time()) - printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(threshold_stucked_module - (int(time.time()) - no_info_modules[curr_queue]))) + "s"]) - else: - printarray3.append([curr_queue, "Stuck or idle, restarting disabled"]) - - - printarray1.sort(key=lambda x: x[0][9:], reverse=False) - printarray2.sort(key=lambda x: x[0][9:], reverse=False) - printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) - printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) - printarray3.insert(0,["Queue", "State"]) - - os.system('clear') - t1 = AsciiTable(printarray1, title="Working queues") - t1.column_max_width(1) - if not t1.ok: - longest_col = t1.column_widths.index(max(t1.column_widths)) - max_length_col = t1.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t1.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t1.table_data[i] = content - - t2 = AsciiTable(printarray2, title="Idling queues") - t2.column_max_width(1) - if not t2.ok: - longest_col = t2.column_widths.index(max(t2.column_widths)) - max_length_col = t2.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t2.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t2.table_data[i] = content - - t3 = AsciiTable(printarray3, title="Not running queues") - t3.column_max_width(1) - - printarray4 = [] - for elem in printarrayGlob: - if elem is not None: - printarray4.append(elem) - - t4 = AsciiTable(printarray4, title="Last actions") - t4.column_max_width(1) - - legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(threshold_stucked_module)]] - legend = AsciiTable(legend_array, title="Legend") - legend.column_max_width(1) - - print legend.table - print '\n' - print t1.table - print '\n' - print t2.table - print '\n' - print t3.table - print '\n' - print t4.table - - if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: - lastTime = datetime.datetime.now() - cleanRedis() - time.sleep(args.refresh) + #If no info since long time, try to kill + if args.autokill == 1: + if int(time.time()) - no_info_modules[curr_queue] > args.treshold: + kill_module(curr_queue, None) + no_info_modules[curr_queue] = int(time.time()) + printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"]) + else: + printarray3.append([curr_queue, "Stuck or idle, restarting disabled"]) + + ## FIXME To add: + ## Button KILL Process using Curses + + printarray1.sort(key=lambda x: x[0][9:], reverse=False) + printarray2.sort(key=lambda x: x[0][9:], reverse=False) + printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) + printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) + printarray3.insert(0,["Queue", "State"]) + + os.system('clear') + t1 = AsciiTable(printarray1, title="Working queues") + t1.column_max_width(1) + if not t1.ok: + longest_col = t1.column_widths.index(max(t1.column_widths)) + max_length_col = t1.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t1.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t1.table_data[i] = content + + t2 = AsciiTable(printarray2, title="Idling queues") + t2.column_max_width(1) + if not t2.ok: + longest_col = t2.column_widths.index(max(t2.column_widths)) + max_length_col = t2.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t2.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t2.table_data[i] = content + + t3 = AsciiTable(printarray3, title="Not running queues") + t3.column_max_width(1) + + printarray4 = [] + for elem in printarrayGlob: + if elem is not None: + printarray4.append(elem) + + t4 = AsciiTable(printarray4, title="Last actions") + t4.column_max_width(1) + + legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]] + legend = AsciiTable(legend_array, title="Legend") + legend.column_max_width(1) + + print legend.table + print '\n' + print t1.table + print '\n' + print t2.table + print '\n' + print t3.table + print '\n' + print t4.table + + if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: + lastTime = datetime.datetime.now() + cleanRedis() + #time.sleep(args.refresh)