mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-30 09:47:17 +00:00
Improved module + Added support of click and keyboard navigation for killing and starting modules
This commit is contained in:
parent
3b8149d840
commit
692f13d18b
1 changed files with 209 additions and 146 deletions
|
@ -8,12 +8,14 @@ from asciimatics.scene import Scene
|
||||||
from asciimatics.screen import Screen
|
from asciimatics.screen import Screen
|
||||||
from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication
|
from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication
|
||||||
from asciimatics.event import Event
|
from asciimatics.event import Event
|
||||||
|
from asciimatics.event import KeyboardEvent, MouseEvent
|
||||||
import sys, os
|
import sys, os
|
||||||
import time, datetime
|
import time, datetime
|
||||||
import argparse, ConfigParser
|
import argparse, ConfigParser
|
||||||
import json
|
import json
|
||||||
import redis
|
import redis
|
||||||
import psutil
|
import psutil
|
||||||
|
from subprocess import PIPE, Popen
|
||||||
|
|
||||||
# CONFIG VARIABLES
|
# CONFIG VARIABLES
|
||||||
kill_retry_threshold = 60 #1m
|
kill_retry_threshold = 60 #1m
|
||||||
|
@ -23,9 +25,17 @@ command_search_name = "ps a -o pid,cmd | grep {}"
|
||||||
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
|
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
|
||||||
|
|
||||||
printarrayGlob = [None]*14
|
printarrayGlob = [None]*14
|
||||||
printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"])
|
|
||||||
lastTimeKillCommand = {}
|
lastTimeKillCommand = {}
|
||||||
TABLES = {"running": [("fetching information...",0)], "idle": [("fetching information...",0)], "notRunning": [("fetching information...",0)], "logs": [("No events recorded yet", 0)]}
|
|
||||||
|
current_selected_value = 0
|
||||||
|
current_selected_queue = ""
|
||||||
|
|
||||||
|
PID_NAME_DICO = {}
|
||||||
|
|
||||||
|
TABLES = {"running": [], "idle": [], "notRunning": [], "logs": [("No events recorded yet", 0)]}
|
||||||
|
TABLES_TITLES = {"running": "", "idle": "", "notRunning": "", "logs": ""}
|
||||||
|
TABLES_PADDING = {"running": [12, 23, 8, 8, 23, 10, 55, 11, 11, 12], "idle": [9, 23, 8, 12, 50], "notRunning": [9, 23, 35], "logs": [15, 23, 8, 50]}
|
||||||
|
|
||||||
QUEUE_STATUS = {}
|
QUEUE_STATUS = {}
|
||||||
CPU_TABLE = {}
|
CPU_TABLE = {}
|
||||||
CPU_OBJECT_TABLE = {}
|
CPU_OBJECT_TABLE = {}
|
||||||
|
@ -63,14 +73,7 @@ class CListBox(ListBox):
|
||||||
self._start_line = max(0, max(self._line - height + 1,
|
self._start_line = max(0, max(self._line - height + 1,
|
||||||
min(self._start_line, self._line)))
|
min(self._start_line, self._line)))
|
||||||
for i, (text, pid) in enumerate(self._options):
|
for i, (text, pid) in enumerate(self._options):
|
||||||
if i == 0:
|
if self._start_line <= i < self._start_line + height:
|
||||||
colour, attr, bg = self._frame.palette["title"]
|
|
||||||
self._frame.canvas.print_at(
|
|
||||||
"{:{width}}".format(text, width=width),
|
|
||||||
self._x + self._offset + dx,
|
|
||||||
self._y + i + dy - self._start_line,
|
|
||||||
colour, attr, bg)
|
|
||||||
elif self._start_line <= i < self._start_line + height:
|
|
||||||
colour, attr, bg = self._pick_colours("field", i == self._line)
|
colour, attr, bg = self._pick_colours("field", i == self._line)
|
||||||
self._frame.canvas.print_at(
|
self._frame.canvas.print_at(
|
||||||
"{:{width}}".format(text, width=width),
|
"{:{width}}".format(text, width=width),
|
||||||
|
@ -91,23 +94,67 @@ class CListBox(ListBox):
|
||||||
colour, attr, queueStatus)
|
colour, attr, queueStatus)
|
||||||
|
|
||||||
|
|
||||||
|
def process_event(self, event):
|
||||||
|
if isinstance(event, KeyboardEvent):
|
||||||
|
if len(self._options) > 0 and event.key_code == Screen.KEY_UP:
|
||||||
|
# Move up one line in text - use value to trigger on_select.
|
||||||
|
self._line = max(0, self._line - 1)
|
||||||
|
self.value = self._options[self._line][1]
|
||||||
|
elif len(self._options) > 0 and event.key_code == Screen.KEY_DOWN:
|
||||||
|
# Move down one line in text - use value to trigger on_select.
|
||||||
|
self._line = min(len(self._options) - 1, self._line + 1)
|
||||||
|
self.value = self._options[self._line][1]
|
||||||
|
elif len(self._options) > 0 and event.key_code == ord(' '):
|
||||||
|
global current_selected_value, current_selected_queue
|
||||||
|
current_selected_value = self.value
|
||||||
|
current_selected_queue = self.queue_name
|
||||||
|
raise NextScene("confirm")
|
||||||
|
else:
|
||||||
|
# Ignore any other key press.
|
||||||
|
return event
|
||||||
|
elif isinstance(event, MouseEvent):
|
||||||
|
# Mouse event - rebase coordinates to Frame context.
|
||||||
|
new_event = self._frame.rebase_event(event)
|
||||||
|
if event.buttons != 0:
|
||||||
|
if (len(self._options) > 0 and
|
||||||
|
self.is_mouse_over(new_event, include_label=False)):
|
||||||
|
# Use property to trigger events.
|
||||||
|
self._line = min(new_event.y - self._y,
|
||||||
|
len(self._options) - 1)
|
||||||
|
self.value = self._options[self._line][1]
|
||||||
|
# If clicked on button <k>, kill the queue
|
||||||
|
if self._x+2 <= new_event.x < self._x+4:
|
||||||
|
if self.queue_name in ["running", "idle"]:
|
||||||
|
kill_module(PID_NAME_DICO[self.value], self.value)
|
||||||
|
else:
|
||||||
|
restart_module(self.value)
|
||||||
|
|
||||||
|
return
|
||||||
|
# Ignore other mouse events.
|
||||||
|
return event
|
||||||
|
else:
|
||||||
|
# Ignore other events
|
||||||
|
return event
|
||||||
|
|
||||||
|
|
||||||
class CLabel(Label):
|
class CLabel(Label):
|
||||||
def __init__(self, label):
|
def __init__(self, label, listTitle=False):
|
||||||
super(Label, self).__init__(None, tab_stop=False)
|
super(Label, self).__init__(None, tab_stop=False)
|
||||||
# Although this is a label, we don't want it to contribute to the layout
|
# Although this is a label, we don't want it to contribute to the layout
|
||||||
# tab calculations, so leave internal `_label` value as None.
|
# tab calculations, so leave internal `_label` value as None.
|
||||||
self._text = label
|
self._text = label
|
||||||
|
self.listTitle = listTitle
|
||||||
|
|
||||||
def set_layout(self, x, y, offset, w, h):
|
def set_layout(self, x, y, offset, w, h):
|
||||||
# Do the usual layout work. then recalculate exact x/w values for the
|
# Do the usual layout work. then recalculate exact x/w values for the
|
||||||
# rendered button.
|
# rendered button.
|
||||||
super(Label, self).set_layout(x, y, offset, w, h)
|
super(Label, self).set_layout(x, y, offset, w, h)
|
||||||
self._x += max(0, (self._w - self._offset - len(self._text)) // 2)
|
self._x += max(0, (self._w - self._offset - len(self._text)) // 2) if not self.listTitle else 0
|
||||||
self._w = min(self._w, len(self._text))
|
self._w = min(self._w, len(self._text))
|
||||||
|
|
||||||
def update(self, frame_no):
|
def update(self, frame_no):
|
||||||
(colour, attr, bg) = self._frame.palette["title"]
|
(colour, attr, bg) = self._frame.palette["title"]
|
||||||
colour = Screen.COLOUR_YELLOW
|
colour = Screen.COLOUR_YELLOW if not self.listTitle else colour
|
||||||
self._frame.canvas.print_at(
|
self._frame.canvas.print_at(
|
||||||
self._text, self._x, self._y, colour, attr, bg)
|
self._text, self._x, self._y, colour, attr, bg)
|
||||||
|
|
||||||
|
@ -116,33 +163,34 @@ class ListView(Frame):
|
||||||
super(ListView, self).__init__(screen,
|
super(ListView, self).__init__(screen,
|
||||||
screen.height,
|
screen.height,
|
||||||
screen.width,
|
screen.width,
|
||||||
on_load=self._reload_list,
|
|
||||||
hover_focus=True,
|
hover_focus=True,
|
||||||
reduce_cpu=True)
|
reduce_cpu=True)
|
||||||
|
|
||||||
self._list_view_run_queue = CListBox(
|
self._list_view_run_queue = CListBox(
|
||||||
"running",
|
"running",
|
||||||
screen.height // 2,
|
screen.height // 2,
|
||||||
[], name="LIST", on_change=self._on_pick)
|
[], name="LIST")
|
||||||
self._list_view_idle_queue = CListBox(
|
self._list_view_idle_queue = CListBox(
|
||||||
"idle",
|
"idle",
|
||||||
screen.height // 2,
|
screen.height // 2,
|
||||||
[], name="LIST", on_change=self._on_pick)
|
[], name="LIST")
|
||||||
self._list_view_noRunning = CListBox(
|
self._list_view_noRunning = CListBox(
|
||||||
"notRunning",
|
"notRunning",
|
||||||
screen.height // 4,
|
screen.height // 5,
|
||||||
[], name="LIST", on_change=self._on_pick)
|
[], name="LIST")
|
||||||
self._list_view_Log = CListBox(
|
self._list_view_Log = CListBox(
|
||||||
"logs",
|
"logs",
|
||||||
screen.height // 4,
|
screen.height // 4,
|
||||||
[], name="LIST", on_change=self._on_pick)
|
[], name="LIST")
|
||||||
self._list_view_Log.disabled = True
|
#self._list_view_Log.disabled = True
|
||||||
|
|
||||||
|
|
||||||
#Running Queues
|
#Running Queues
|
||||||
layout = Layout([100])
|
layout = Layout([100])
|
||||||
self.add_layout(layout)
|
self.add_layout(layout)
|
||||||
text_rq = CLabel("Running Queues")
|
text_rq = CLabel("Running Queues")
|
||||||
layout.add_widget(text_rq)
|
layout.add_widget(text_rq)
|
||||||
|
layout.add_widget(CLabel(TABLES_TITLES["running"], listTitle=True))
|
||||||
layout.add_widget(self._list_view_run_queue)
|
layout.add_widget(self._list_view_run_queue)
|
||||||
layout.add_widget(Divider())
|
layout.add_widget(Divider())
|
||||||
|
|
||||||
|
@ -150,36 +198,86 @@ class ListView(Frame):
|
||||||
layout2 = Layout([1,1])
|
layout2 = Layout([1,1])
|
||||||
self.add_layout(layout2)
|
self.add_layout(layout2)
|
||||||
text_iq = CLabel("Idling Queues")
|
text_iq = CLabel("Idling Queues")
|
||||||
layout2.add_widget(text_iq)
|
layout2.add_widget(text_iq, 0)
|
||||||
|
layout2.add_widget(CLabel(TABLES_TITLES["idle"], listTitle=True), 0)
|
||||||
layout2.add_widget(self._list_view_idle_queue, 0)
|
layout2.add_widget(self._list_view_idle_queue, 0)
|
||||||
#Non Running Queues
|
#Non Running Queues
|
||||||
text_nq = CLabel("No Running Queues")
|
text_nq = CLabel("No Running Queues")
|
||||||
layout2.add_widget(text_nq, 1)
|
layout2.add_widget(text_nq, 1)
|
||||||
|
layout2.add_widget(CLabel(TABLES_TITLES["notRunning"], listTitle=True), 1)
|
||||||
layout2.add_widget(self._list_view_noRunning, 1)
|
layout2.add_widget(self._list_view_noRunning, 1)
|
||||||
layout2.add_widget(Divider(), 1)
|
layout2.add_widget(Divider(), 1)
|
||||||
#Log
|
#Log
|
||||||
text_l = CLabel("Logs")
|
text_l = CLabel("Logs")
|
||||||
layout2.add_widget(text_l, 1)
|
layout2.add_widget(text_l, 1)
|
||||||
|
layout2.add_widget(CLabel(TABLES_TITLES["logs"], listTitle=True), 1)
|
||||||
layout2.add_widget(self._list_view_Log, 1)
|
layout2.add_widget(self._list_view_Log, 1)
|
||||||
|
|
||||||
self.fix()
|
self.fix()
|
||||||
self._on_pick()
|
|
||||||
|
|
||||||
def _on_pick(self):
|
|
||||||
return
|
|
||||||
|
|
||||||
def _reload_list(self):
|
|
||||||
self._list_view_run_queue = [(time.time(), 123)]
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _quit():
|
def _quit():
|
||||||
raise StopApplication("User pressed quit")
|
raise StopApplication("User pressed quit")
|
||||||
|
|
||||||
|
class Confirm(Frame):
|
||||||
|
def __init__(self, screen):
|
||||||
|
super(Confirm, self).__init__(screen,
|
||||||
|
screen.height * 1 // 8,
|
||||||
|
screen.width * 1 // 4,
|
||||||
|
hover_focus=True,
|
||||||
|
on_load=self._setValue,
|
||||||
|
title="Confirm action",
|
||||||
|
reduce_cpu=True)
|
||||||
|
|
||||||
|
# Create the form for displaying the list of contacts.
|
||||||
|
layout = Layout([100], fill_frame=True)
|
||||||
|
self.add_layout(layout)
|
||||||
|
self.label = CLabel("Confirm {} module {} {}?")
|
||||||
|
layout.add_widget(self.label)
|
||||||
|
layout2 = Layout([1,1])
|
||||||
|
self.add_layout(layout2)
|
||||||
|
layout2.add_widget(Button("Ok", self._ok), 0)
|
||||||
|
layout2.add_widget(Button("Cancel", self._cancel), 1)
|
||||||
|
|
||||||
|
self.fix()
|
||||||
|
|
||||||
|
def _ok(self):
|
||||||
|
global current_selected_value, current_selected_queue
|
||||||
|
if current_selected_queue in ["running", "idle"]:
|
||||||
|
kill_module(PID_NAME_DICO[current_selected_value], current_selected_value)
|
||||||
|
else:
|
||||||
|
restart_module(current_selected_value)
|
||||||
|
current_selected_value = 0
|
||||||
|
current_selected_value = 0
|
||||||
|
self.label._text = "Confirm {} module {} {}?"
|
||||||
|
raise NextScene("dashboard")
|
||||||
|
|
||||||
|
def _cancel(self):
|
||||||
|
global current_selected_value
|
||||||
|
current_selected_value = 0
|
||||||
|
self.label._text = "Confirm {} module {} {}?"
|
||||||
|
raise NextScene("dashboard")
|
||||||
|
|
||||||
|
def _setValue(self):
|
||||||
|
global current_selected_value, current_selected_queue
|
||||||
|
if current_selected_queue in ["running", "idle"]:
|
||||||
|
action = "KILL"
|
||||||
|
modulename = PID_NAME_DICO[current_selected_value]
|
||||||
|
pid = current_selected_value
|
||||||
|
else:
|
||||||
|
action = "START"
|
||||||
|
modulename = current_selected_value
|
||||||
|
pid = ""
|
||||||
|
self.label._text = self.label._text.format(action, modulename, pid)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def demo(screen):
|
def demo(screen):
|
||||||
LV = ListView(screen)
|
dashboard = ListView(screen)
|
||||||
|
confirm = Confirm(screen)
|
||||||
scenes = [
|
scenes = [
|
||||||
Scene([LV], -1)
|
Scene([dashboard], -1, name="dashboard"),
|
||||||
|
Scene([confirm], -1, name="confirm"),
|
||||||
]
|
]
|
||||||
|
|
||||||
# screen.play(scenes)
|
# screen.play(scenes)
|
||||||
|
@ -187,14 +285,16 @@ def demo(screen):
|
||||||
time_cooldown = time.time()
|
time_cooldown = time.time()
|
||||||
global TABLES
|
global TABLES
|
||||||
while True:
|
while True:
|
||||||
LV._update(None)
|
|
||||||
screen.draw_next_frame()
|
|
||||||
if time.time() - time_cooldown > args.refresh:
|
if time.time() - time_cooldown > args.refresh:
|
||||||
|
cleanRedis()
|
||||||
for key, val in fetchQueueData().iteritems():
|
for key, val in fetchQueueData().iteritems():
|
||||||
TABLES[key] = val
|
TABLES[key] = val
|
||||||
TABLES["logs"] = format_logs(printarrayGlob)
|
TABLES["logs"] = format_string(printarrayGlob, TABLES_PADDING["logs"])
|
||||||
screen.refresh()
|
if current_selected_value == 0:
|
||||||
|
dashboard._update(None)
|
||||||
|
screen.refresh()
|
||||||
time_cooldown = time.time()
|
time_cooldown = time.time()
|
||||||
|
screen.draw_next_frame()
|
||||||
time.sleep(0.02)
|
time.sleep(0.02)
|
||||||
|
|
||||||
|
|
||||||
|
@ -211,7 +311,7 @@ def clearRedisModuleInfo():
|
||||||
for k in server.keys("MODULE_*"):
|
for k in server.keys("MODULE_*"):
|
||||||
server.delete(k)
|
server.delete(k)
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, "*", "-", "Cleared redis module info"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], "*", "-", "Cleared redis module info"], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
|
|
||||||
def cleanRedis():
|
def cleanRedis():
|
||||||
|
@ -226,21 +326,29 @@ def cleanRedis():
|
||||||
flag_pid_valid = True
|
flag_pid_valid = True
|
||||||
|
|
||||||
if not flag_pid_valid:
|
if not flag_pid_valid:
|
||||||
print flag_pid_valid, 'cleaning', pid, 'in', k
|
#print flag_pid_valid, 'cleaning', pid, 'in', k
|
||||||
server.srem(k, pid)
|
server.srem(k, pid)
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + k], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
#time.sleep(5)
|
#time.sleep(5)
|
||||||
|
|
||||||
|
def restart_module(module):
|
||||||
|
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
||||||
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, "?", "Restarted"], 0))
|
||||||
|
printarrayGlob.pop()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def kill_module(module, pid):
|
def kill_module(module, pid):
|
||||||
print ''
|
#print ''
|
||||||
print '-> trying to kill module:', module
|
#print '-> trying to kill module:', module
|
||||||
|
|
||||||
if pid is None:
|
if pid is None:
|
||||||
print 'pid was None'
|
#print 'pid was None'
|
||||||
printarrayGlob.insert(1, [0, module, pid, "PID was None"])
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "PID was None"], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
pid = getPid(module)
|
pid = getPid(module)
|
||||||
else: #Verify that the pid is at least in redis
|
else: #Verify that the pid is at least in redis
|
||||||
|
@ -251,54 +359,48 @@ def kill_module(module, pid):
|
||||||
if pid is not None:
|
if pid is not None:
|
||||||
try:
|
try:
|
||||||
#os.kill(pid, signal.SIGUSR1)
|
#os.kill(pid, signal.SIGUSR1)
|
||||||
p = psutil.Process(pid)
|
p = psutil.Process(int(pid))
|
||||||
p.terminate()
|
p.terminate()
|
||||||
except Exception:
|
except Exception as e:
|
||||||
print pid, 'already killed'
|
#print pid, 'already killed'
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Already killed"], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
return
|
return
|
||||||
time.sleep(1)
|
time.sleep(0.2)
|
||||||
if getPid(module) is None:
|
if not p.is_running():
|
||||||
print module, 'has been killed'
|
#print module, 'has been killed'
|
||||||
print 'restarting', module, '...'
|
#print 'restarting', module, '...'
|
||||||
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
|
||||||
printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"])
|
|
||||||
printarrayGlob.pop()
|
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
|
restart_module(module)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
print 'killing failed, retrying...'
|
#print 'killing failed, retrying...'
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Killing #1 failed."], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
|
|
||||||
time.sleep(1)
|
|
||||||
#os.kill(pid, signal.SIGUSR1)
|
#os.kill(pid, signal.SIGUSR1)
|
||||||
p = psutil.Process(pid)
|
#time.sleep(1)
|
||||||
p.terminate()
|
p.terminate()
|
||||||
time.sleep(1)
|
if not p.is_running():
|
||||||
if getPid(module) is None:
|
#print module, 'has been killed'
|
||||||
print module, 'has been killed'
|
#print 'restarting', module, '...'
|
||||||
print 'restarting', module, '...'
|
|
||||||
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
|
||||||
printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"])
|
|
||||||
printarrayGlob.pop()
|
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
|
restart_module(module)
|
||||||
else:
|
else:
|
||||||
print 'killing failed!'
|
#print 'killing failed!'
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Killing failed!"], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
else:
|
else:
|
||||||
print 'Module does not exist'
|
#print 'Module does not exist'
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
||||||
printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"])
|
printarrayGlob.insert(0, ([str(inst_time).split(' ')[1], module, pid, "Killing failed, module not found"], 0))
|
||||||
printarrayGlob.pop()
|
printarrayGlob.pop()
|
||||||
#time.sleep(5)
|
#time.sleep(5)
|
||||||
cleanRedis()
|
cleanRedis()
|
||||||
|
@ -347,112 +449,68 @@ def fetchQueueData():
|
||||||
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
||||||
CPU_TABLE[moduleNum].insert(1, cpu_percent)
|
CPU_TABLE[moduleNum].insert(1, cpu_percent)
|
||||||
cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum])
|
cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum])
|
||||||
|
if len(CPU_TABLE[moduleNum]) > args.refresh*10:
|
||||||
|
CPU_TABLE[moduleNum].pop()
|
||||||
|
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
|
||||||
except KeyError:
|
except KeyError:
|
||||||
CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum))
|
try:
|
||||||
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum))
|
||||||
CPU_TABLE[moduleNum] = []
|
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
||||||
cpu_avg = cpu_percent
|
CPU_TABLE[moduleNum] = []
|
||||||
if len(CPU_TABLE[moduleNum]) > args.refresh*10:
|
cpu_avg = cpu_percent
|
||||||
CPU_TABLE[moduleNum].pop()
|
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
|
||||||
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
|
except psutil.NoSuchProcess:
|
||||||
|
cpu_percent = 0
|
||||||
|
cpu_avg = cpu_percent
|
||||||
|
mem_percent = 0
|
||||||
|
|
||||||
array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%", "{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) )
|
array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%", "{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) )
|
||||||
|
|
||||||
else:
|
else:
|
||||||
printarray2.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
|
printarray2.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
|
||||||
|
PID_NAME_DICO[moduleNum] = str(queue)
|
||||||
array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True)
|
array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True)
|
||||||
for e in array_module_type:
|
for e in array_module_type:
|
||||||
printarray1.append(e)
|
printarray1.append(e)
|
||||||
|
|
||||||
for curr_queue in module_file_array:
|
for curr_queue in module_file_array:
|
||||||
if curr_queue not in all_queue:
|
if curr_queue not in all_queue:
|
||||||
printarray3.append( ([" <S> ", curr_queue, "Not running"], len(printarray3)+1) )
|
printarray3.append( ([" <S> ", curr_queue, "Not running by default"], curr_queue) )
|
||||||
else:
|
else:
|
||||||
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
|
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
|
||||||
if curr_queue not in no_info_modules:
|
if curr_queue not in no_info_modules:
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
no_info_modules[curr_queue] = int(time.time())
|
||||||
printarray3.append( ([" <S> ", curr_queue, "No data"], len(printarray3)+1) )
|
printarray3.append( ([" <S> ", curr_queue, "No data"], curr_queue) )
|
||||||
else:
|
else:
|
||||||
#If no info since long time, try to kill
|
#If no info since long time, try to kill
|
||||||
if args.autokill == 1:
|
if args.autokill == 1:
|
||||||
if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
|
if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
|
||||||
kill_module(curr_queue, None)
|
kill_module(curr_queue, None)
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
no_info_modules[curr_queue] = int(time.time())
|
||||||
printarray3.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], len(printarray3)+1) )
|
printarray3.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], curr_queue) )
|
||||||
else:
|
else:
|
||||||
printarray3.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], len(printarray3)+1) )
|
printarray3.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], curr_queue) )
|
||||||
|
|
||||||
## FIXME To add:
|
|
||||||
## Button KILL Process using Curses
|
|
||||||
|
|
||||||
printarray1.sort(key=lambda x: x[0], reverse=False)
|
printarray1.sort(key=lambda x: x[0], reverse=False)
|
||||||
printarray2.sort(key=lambda x: x[0], reverse=False)
|
printarray2.sort(key=lambda x: x[0], reverse=False)
|
||||||
printarray1.insert(0,([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"], 0) )
|
|
||||||
printarray2.insert(0,([" Action", "Queue", "PID", "Idle Time", "Last paste hash"], 0) )
|
|
||||||
printarray3.insert(0,([" Action", "Queue", "State"], 0) )
|
|
||||||
|
|
||||||
padding_row = [12, 23, 8,
|
printstring1 = format_string(printarray1, TABLES_PADDING["running"])
|
||||||
8, 23, 10,
|
printstring2 = format_string(printarray2, TABLES_PADDING["idle"])
|
||||||
55, 11, 11, 12]
|
printstring3 = format_string(printarray3, TABLES_PADDING["notRunning"])
|
||||||
printstring1 = []
|
|
||||||
for row in printarray1:
|
|
||||||
the_array = row[0]
|
|
||||||
the_pid = row[1]
|
|
||||||
text=""
|
|
||||||
for ite, elem in enumerate(the_array):
|
|
||||||
if len(elem) > padding_row[ite]:
|
|
||||||
text += "*" + elem[-padding_row[ite]+6:]
|
|
||||||
padd_off = " "*5
|
|
||||||
else:
|
|
||||||
text += elem
|
|
||||||
padd_off = " "*0
|
|
||||||
text += (padding_row[ite] - len(elem))*" " + padd_off
|
|
||||||
printstring1.append( (text, the_pid) )
|
|
||||||
|
|
||||||
padding_row = [9, 23, 8,
|
|
||||||
12, 50]
|
|
||||||
printstring2 = []
|
|
||||||
for row in printarray2:
|
|
||||||
the_array = row[0]
|
|
||||||
the_pid = row[1]
|
|
||||||
text=""
|
|
||||||
for ite, elem in enumerate(the_array):
|
|
||||||
if len(elem) > padding_row[ite]:
|
|
||||||
text += "*" + elem[-padding_row[ite]+6:]
|
|
||||||
padd_off = " "*5
|
|
||||||
else:
|
|
||||||
text += elem
|
|
||||||
padd_off = " "*0
|
|
||||||
text += (padding_row[ite] - len(elem))*" " + padd_off
|
|
||||||
printstring2.append( (text, the_pid) )
|
|
||||||
|
|
||||||
padding_row = [9, 23, 35]
|
|
||||||
printstring3 = []
|
|
||||||
for row in printarray3:
|
|
||||||
the_array = row[0]
|
|
||||||
the_pid = row[1]
|
|
||||||
text=""
|
|
||||||
for ite, elem in enumerate(the_array):
|
|
||||||
if len(elem) > padding_row[ite]:
|
|
||||||
text += "*" + elem[-padding_row[ite]+6:]
|
|
||||||
padd_off = " "*5
|
|
||||||
else:
|
|
||||||
text += elem
|
|
||||||
padd_off = " "*0
|
|
||||||
text += (padding_row[ite] - len(elem))*" " + padd_off
|
|
||||||
printstring3.append( (text, the_pid) )
|
|
||||||
|
|
||||||
return {"running": printstring1, "idle": printstring2, "notRunning": printstring3}
|
return {"running": printstring1, "idle": printstring2, "notRunning": printstring3}
|
||||||
|
|
||||||
def format_logs(logs):
|
def format_string(tab, padding_row):
|
||||||
printstring4 = []
|
printstring = []
|
||||||
padding_row = [12, 23, 8, 50]
|
for row in tab:
|
||||||
text=""
|
|
||||||
|
|
||||||
for row in logs:
|
|
||||||
if row is None:
|
if row is None:
|
||||||
continue
|
continue
|
||||||
for ite, elem in enumerate(row):
|
the_array = row[0]
|
||||||
|
the_pid = row[1]
|
||||||
|
|
||||||
|
text=""
|
||||||
|
for ite, elem in enumerate(the_array):
|
||||||
if len(elem) > padding_row[ite]:
|
if len(elem) > padding_row[ite]:
|
||||||
text += "*" + elem[-padding_row[ite]+6:]
|
text += "*" + elem[-padding_row[ite]+6:]
|
||||||
padd_off = " "*5
|
padd_off = " "*5
|
||||||
|
@ -460,15 +518,15 @@ def format_logs(logs):
|
||||||
text += elem
|
text += elem
|
||||||
padd_off = " "*0
|
padd_off = " "*0
|
||||||
text += (padding_row[ite] - len(elem))*" " + padd_off
|
text += (padding_row[ite] - len(elem))*" " + padd_off
|
||||||
printstring4.append( (text, len(printstring4)+1) )
|
printstring.append( (text, the_pid) )
|
||||||
return printstring4
|
return printstring
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
|
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
|
||||||
parser.add_argument('-r', '--refresh', type=int, required=False, default=2, help='Refresh rate')
|
parser.add_argument('-r', '--refresh', type=int, required=False, default=5, help='Refresh rate')
|
||||||
parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
|
parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
|
||||||
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
|
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
|
||||||
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
|
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
|
||||||
|
@ -502,8 +560,13 @@ if __name__ == "__main__":
|
||||||
for line in module_file:
|
for line in module_file:
|
||||||
module_file_array.add(line[:-1])
|
module_file_array.add(line[:-1])
|
||||||
|
|
||||||
#cleanRedis()
|
cleanRedis()
|
||||||
|
|
||||||
|
|
||||||
|
TABLES_TITLES["running"] = format_string([([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"],0)], TABLES_PADDING["running"])[0][0]
|
||||||
|
TABLES_TITLES["idle"] = format_string([([" Action", "Queue", "PID", "Idle Time", "Last paste hash"],0)], TABLES_PADDING["idle"])[0][0]
|
||||||
|
TABLES_TITLES["notRunning"] = format_string([([" Action", "Queue", "State"],0)], TABLES_PADDING["notRunning"])[0][0]
|
||||||
|
TABLES_TITLES["logs"] = format_string([(["Time", "Module", "PID", "Info"],0)], TABLES_PADDING["logs"])[0][0]
|
||||||
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
|
Loading…
Reference in a new issue