Merge branch 'master' of github.com:CIRCL/AIL-framework

bin/BrowseWarningPaste.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"
 
-    config_section = 'Browse_warning_paste'
+    config_section = 'BrowseWarningPaste'
 
     p = Process(config_section)
 

bin/Helper.py

@@ -119,13 +119,7 @@ class Process(object):
             port=self.config.get('RedisPubSub', 'port'),
             db=self.config.get('RedisPubSub', 'db'))
 
-        self.moduleNum = 1
-        for i in range(1, 50):
-            curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i))
-            if curr_num is None:
-                self.moduleNum = i
-                break
-
+        self.moduleNum = os.getpid()
 
 
     def populate_set_in(self):
@@ -158,12 +152,14 @@ class Process(object):
                     path = "?"
                 value = str(timestamp) + ", " + path
                 self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+                self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
                 return message
 
             except:
                 path = "?"
                 value = str(timestamp) + ", " + path
                 self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+                self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
                 return message
 
     def populate_set_out(self, msg, channel=None):

bin/LAUNCH.sh

@@ -158,7 +158,7 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
     sleep 0.1
-    screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+    screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
 

bin/ModuleInformation.py

@@ -29,6 +29,7 @@ import textwrap
 threshold_stucked_module = 60*60*1 #1 hour
 log_filename = "../logs/moduleInfo.log"
 command_search_pid = "ps a -o pid,cmd | grep {}"
+command_search_name = "ps a -o pid,cmd | grep {}"
 command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
 
 
@@ -45,6 +46,23 @@ def clearRedisModuleInfo():
     for k in server.keys("MODULE_*"):
         server.delete(k)
 
+def cleanRedis():
+    for k in server.keys("MODULE_TYPE_*"):
+        moduleName = k[12:].split('_')[0]
+        for pid in server.smembers(k):
+            flag_pid_valid = False
+            proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
+            for line in proc.stdout:
+                splittedLine = line.split()
+                if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine:
+                    flag_pid_valid = True
+
+            if not flag_pid_valid:
+                print flag_pid_valid, 'cleaning', pid, 'in', k
+                server.srem(k, pid)
+                time.sleep(5)
+
+
 def kill_module(module):
     print ''
     print '-> trying to kill module:', module
@@ -76,8 +94,8 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
     parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
-    parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
-    parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)')
+    parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
+    parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
 
     args = parser.parse_args()
 
@@ -99,6 +117,7 @@ if __name__ == "__main__":
     if args.clear == 1:
         clearRedisModuleInfo()
 
+    lastTime = datetime.datetime.now()
 
     module_file_array = set()
     with open('../doc/all_modules.txt', 'r') as module_file:
@@ -108,20 +127,15 @@ if __name__ == "__main__":
         while True:
 
             all_queue = set()
-            curr_range = 50
             printarray1 = []
             printarray2 = []
             printarray3 = []
             for queue, card in server.hgetall("queues").iteritems():
                 all_queue.add(queue)
                 key = "MODULE_" + queue + "_"
-                for i in range(1, 50):
-                    curr_num = server.get("MODULE_"+ queue + "_" + str(i))
-                    if curr_num is None:
-                        curr_range = i
-                        break
+                keySet = "MODULE_TYPE_" + queue
 
-                for moduleNum in range(1, curr_range):
+                for moduleNum in server.smembers(keySet):
                     value = server.get(key + str(moduleNum))
                     if value is not None:
                         timestamp, path = value.split(", ")
@@ -147,8 +161,8 @@ if __name__ == "__main__":
 
             printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
             printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
-            printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
-            printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
+            printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
+            printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
             printarray3.insert(0,["Queue", "State"])
 
             os.system('clear')
@@ -195,4 +209,7 @@ if __name__ == "__main__":
             print '\n'
             print t3.table
 
+            if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: 
+                lastTime = datetime.datetime.now()
+                cleanRedis()
             time.sleep(args.refresh)

bin/packages/modules.cfg

@@ -63,7 +63,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate
 [ModuleStats]
 subscribe = Redis_ModuleStats
 
-[Browse_warning_paste]
+[BrowseWarningPaste]
 subscribe = Redis_BrowseWarningPaste
 
 #[send_to_queue]

var/www/Flask_server.py

@@ -81,19 +81,13 @@ def event_stream():
 
 def get_queues(r):
     # We may want to put the llen in a pipeline to do only one query.
-    data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()]
     newData = []
-
-    curr_range = 50
-    for queue, card in data:
+    for queue, card in r.hgetall("queues").iteritems():
         key = "MODULE_" + queue + "_"
-        for i in range(1, 50):
-            curr_num = r.get("MODULE_"+ queue + "_" + str(i))
-            if curr_num is None:
-                curr_range = i
-                break
+        keySet = "MODULE_TYPE_" + queue
 
-        for moduleNum in range(1, curr_range):
+        for moduleNum in r.smembers(keySet):
+    
             value = r.get(key + str(moduleNum))
             if value is not None:
                 timestamp, path = value.split(", ")

var/www/static/js/indexjavascript.js

@@ -207,7 +207,7 @@ function create_queue_table() {
     table.appendChild(tableHead);
     table.appendChild(tableBody);
     var heading = new Array();
-    heading[0] = "Queue Name"
+    heading[0] = "Queue Name.PID"
     heading[1] = "Amount"
     var tr = document.createElement('TR');
     tableHead.appendChild(tr);
@@ -244,12 +244,8 @@ function create_queue_table() {
     Tablediv.appendChild(table);
 }
 
-$(document).ready(function () {
-    if (typeof glob_tabvar == "undefined")
-        location.reload();
-    if (typeof glob_tabvar.row1 == "undefined")
-        location.reload();
 
+function load_queues() {
     var data = [];
     var data2 = [];
     var tmp_tab = [];
@@ -259,13 +255,17 @@ $(document).ready(function () {
     var x = new Date();
 
     for (i = 0; i < glob_tabvar.row1.length; i++){
-        if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
-            tmp_tab2.push(0);
-            curves_labels2.push(glob_tabvar.row1[i][0]);
+        if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
+            if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                tmp_tab2.push(0);
+                curves_labels2.push(glob_tabvar.row1[i][0].split(".")[0]);
+            }
         }
         else {
-            tmp_tab.push(0);
-            curves_labels.push(glob_tabvar.row1[i][0]);
+            if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                tmp_tab.push(0);
+                curves_labels.push(glob_tabvar.row1[i][0].split(".")[0]);
+            }
         }
     }
     tmp_tab.unshift(x);
@@ -324,19 +324,29 @@ $(document).ready(function () {
                     update_values();
 
                     if($('#button-toggle-queues').prop('checked')){
+                        $("#queue-color-legend").show();
                         create_queue_table();
                     }
                     else{
                         $("#queueing").html('');
+                        $("#queue-color-legend").hide();
                     }
 
 
+                    queues_pushed = []
                     for (i = 0; i < (glob_tabvar.row1).length; i++){
-                        if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
-                            tmp_values2.push(glob_tabvar.row1[i][1]);
+                        if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
+                            if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                                queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+                                tmp_values2.push(parseInt(glob_tabvar.row1[i][1]));
+                            }
                         }
                         else {
-                            tmp_values.push(glob_tabvar.row1[i][1]);
+                            if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                                queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+                                tmp_values.push(parseInt(glob_tabvar.row1[i][1]));
+                            }
+                            
                         }
                     }
                     tmp_values.unshift(x);
@@ -375,7 +385,19 @@ $(document).ready(function () {
       // something went wrong, hide the canvas container
       document.getElementById('myCanvasContainer').style.display = 'none';
     }
+}
 
+function manage_undefined() {
+    if (typeof glob_tabvar == "undefined")
+        setTimeout(function() { if (typeof glob_tabvar == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+    else if (typeof glob_tabvar.row1 == "undefined")
+        setTimeout(function() { if (typeof glob_tabvar.row1 == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+    else
+        load_queues();
+}
+
+$(document).ready(function () {
+    manage_undefined();
 });
 
 

var/www/templates/index.html

@@ -66,7 +66,18 @@
                   </label>
                   <strong style="top: 3px; position: relative;">Display queues</strong>
               <div>
-              <div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
+              <div>
+                  <table id="queue-color-legend" class="table">
+                  <thead>
+                  </thead>
+                  <tbody>
+                      <tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
+                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #faf2cc;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
+                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stucked queues</td></tr>
+                  </tbody>
+                  </table>
+              </div>
+              <div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
              <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
           </div>
           <!-- /.navbar-static-side -->

var/www/templates/terms_management.html

@@ -12,7 +12,6 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>

var/www/templates/terms_plot_top.html

@@ -12,7 +12,6 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>

var/www/update_thirdparty.sh

@@ -39,12 +39,17 @@ wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
 
-#Ressources for sparkline and canvasJS
+#Ressources for sparkline and canvasJS and slider
 wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
+mkdir temp
 wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
 unzip temp/canvasjs.zip -d temp/
-mkdir temp
 mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
+
+wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
+unzip temp/jquery-ui.zip -d temp/
+mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js
+mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/js/jquery-ui.min.css
 rm -rf temp
 
 mkdir -p ./static/image