diff --git a/bin/lib/queues_modules.py b/bin/lib/queues_modules.py new file mode 100755 index 00000000..d9999140 --- /dev/null +++ b/bin/lib/queues_modules.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +r_serv_queues = config_loader.get_redis_conn("Redis_Queues") +config_loader = None + +def get_all_queues_name(): + return r_serv_queues.hkeys('queues') + +def get_all_queues_dict_with_nb_elem(): + return r_serv_queues.hgetall('queues') + +def get_all_queues_with_sorted_nb_elem(): + res = r_serv_queues.hgetall('queues') + res = sorted(res.items()) + return res + +def get_module_pid_by_queue_name(queue_name): + return r_serv_queues.smembers('MODULE_TYPE_{}'.format(queue_name)) + +# # TODO: remove last msg part +def get_module_last_process_start_time(queue_name, module_pid): + res = r_serv_queues.get('MODULE_{}_{}'.format(queue_name, module_pid)) + if res: + return res.split(',')[0] + return None + +def get_module_last_msg(queue_name, module_pid): + return r_serv_queues.get('MODULE_{}_{}_PATH'.format(queue_name, module_pid)) + +def get_all_modules_queues_stats(): + all_modules_queues_stats = [] + for queue_name, nb_elem_queue in get_all_queues_with_sorted_nb_elem(): + l_module_pid = get_module_pid_by_queue_name(queue_name) + for module_pid in l_module_pid: + last_process_start_time = get_module_last_process_start_time(queue_name, module_pid) + if last_process_start_time: + last_process_start_time = datetime.datetime.fromtimestamp(int(last_process_start_time)) + seconds = int((datetime.datetime.now() - last_process_start_time).total_seconds()) + else: + seconds = 0 + all_modules_queues_stats.append((queue_name, nb_elem_queue, seconds, module_pid)) + return all_modules_queues_stats + +if __name__ == '__main__': + res = get_all_modules_queues_stats() + print(res) diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 2624ee74..0091df5a 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -6,6 +6,7 @@ ''' import json import os +import sys import datetime import time import flask @@ -17,6 +18,9 @@ from flask import Flask, render_template, jsonify, request, Blueprint, url_for from Role_Manager import login_admin, login_analyst, login_read_only from flask_login import login_required +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import queues_modules + # ============ VARIABLES ============ import Flask_config @@ -52,27 +56,7 @@ def event_stream(): def get_queues(r): # We may want to put the llen in a pipeline to do only one query. - newData = [] - for queue, card in r.hgetall("queues").items(): - - key = "MODULE_" + queue + "_" - keySet = "MODULE_TYPE_" + queue - - for moduleNum in r.smembers(keySet): - - value = r.get(key + str(moduleNum)) - - if value is not None: - timestamp, path = value.split(", ") - if timestamp is not None: - startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) - processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] - seconds = int((datetime.datetime.now() - startTime_readable).total_seconds()) - newData.append( (queue, card, seconds, moduleNum) ) - else: - newData.append( (queue, cards, 0, moduleNum) ) - - return newData + return queues_modules.get_all_modules_queues_stats() def get_date_range(date_from, num_day): date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2))