diff --git a/bin/Browse_warning_paste.py b/bin/BrowseWarningPaste.py
similarity index 97%
rename from bin/Browse_warning_paste.py
rename to bin/BrowseWarningPaste.py
index 49444979..01704902 100755
--- a/bin/Browse_warning_paste.py
+++ b/bin/BrowseWarningPaste.py
@@ -24,7 +24,7 @@ if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
- config_section = 'Browse_warning_paste'
+ config_section = 'BrowseWarningPaste'
p = Process(config_section)
diff --git a/bin/Helper.py b/bin/Helper.py
index e7338ceb..05b73bf3 100755
--- a/bin/Helper.py
+++ b/bin/Helper.py
@@ -119,13 +119,7 @@ class Process(object):
port=self.config.get('RedisPubSub', 'port'),
db=self.config.get('RedisPubSub', 'db'))
- self.moduleNum = 1
- for i in range(1, 50):
- curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i))
- if curr_num is None:
- self.moduleNum = i
- break
-
+ self.moduleNum = os.getpid()
def populate_set_in(self):
@@ -158,12 +152,14 @@ class Process(object):
path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+ self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
return message
except:
path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+ self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
return message
def populate_set_out(self, msg, channel=None):
diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh
index b50a75d6..d7c31472 100755
--- a/bin/LAUNCH.sh
+++ b/bin/LAUNCH.sh
@@ -158,7 +158,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1
- screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+ screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py
index d783418f..1538f57a 100755
--- a/bin/ModuleInformation.py
+++ b/bin/ModuleInformation.py
@@ -29,6 +29,7 @@ import textwrap
threshold_stucked_module = 60*60*1 #1 hour
log_filename = "../logs/moduleInfo.log"
command_search_pid = "ps a -o pid,cmd | grep {}"
+command_search_name = "ps a -o pid,cmd | grep {}"
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
@@ -45,6 +46,23 @@ def clearRedisModuleInfo():
for k in server.keys("MODULE_*"):
server.delete(k)
+def cleanRedis():
+ for k in server.keys("MODULE_TYPE_*"):
+ moduleName = k[12:].split('_')[0]
+ for pid in server.smembers(k):
+ flag_pid_valid = False
+ proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
+ for line in proc.stdout:
+ splittedLine = line.split()
+ if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine:
+ flag_pid_valid = True
+
+ if not flag_pid_valid:
+ print flag_pid_valid, 'cleaning', pid, 'in', k
+ server.srem(k, pid)
+ time.sleep(5)
+
+
def kill_module(module):
print ''
print '-> trying to kill module:', module
@@ -76,8 +94,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
- parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
- parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)')
+ parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
+ parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
args = parser.parse_args()
@@ -99,6 +117,7 @@ if __name__ == "__main__":
if args.clear == 1:
clearRedisModuleInfo()
+ lastTime = datetime.datetime.now()
module_file_array = set()
with open('../doc/all_modules.txt', 'r') as module_file:
@@ -108,20 +127,15 @@ if __name__ == "__main__":
while True:
all_queue = set()
- curr_range = 50
printarray1 = []
printarray2 = []
printarray3 = []
for queue, card in server.hgetall("queues").iteritems():
all_queue.add(queue)
key = "MODULE_" + queue + "_"
- for i in range(1, 50):
- curr_num = server.get("MODULE_"+ queue + "_" + str(i))
- if curr_num is None:
- curr_range = i
- break
+ keySet = "MODULE_TYPE_" + queue
- for moduleNum in range(1, curr_range):
+ for moduleNum in server.smembers(keySet):
value = server.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")
@@ -147,8 +161,8 @@ if __name__ == "__main__":
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
- printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
- printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
+ printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
+ printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
printarray3.insert(0,["Queue", "State"])
os.system('clear')
@@ -195,4 +209,7 @@ if __name__ == "__main__":
print '\n'
print t3.table
+ if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
+ lastTime = datetime.datetime.now()
+ cleanRedis()
time.sleep(args.refresh)
diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg
index 0243038a..782536c5 100644
--- a/bin/packages/modules.cfg
+++ b/bin/packages/modules.cfg
@@ -63,7 +63,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate
[ModuleStats]
subscribe = Redis_ModuleStats
-[Browse_warning_paste]
+[BrowseWarningPaste]
subscribe = Redis_BrowseWarningPaste
#[send_to_queue]
diff --git a/doc/screenshots/terms-plot.png b/doc/screenshots/terms-plot.png
index 32647906..4d33466d 100644
Binary files a/doc/screenshots/terms-plot.png and b/doc/screenshots/terms-plot.png differ
diff --git a/doc/screenshots/terms-top.png b/doc/screenshots/terms-top.png
index e833ac35..fc75aed8 100644
Binary files a/doc/screenshots/terms-top.png and b/doc/screenshots/terms-top.png differ
diff --git a/doc/screenshots/trending-module.png b/doc/screenshots/trending-module.png
index 1330ead8..48304059 100644
Binary files a/doc/screenshots/trending-module.png and b/doc/screenshots/trending-module.png differ
diff --git a/doc/screenshots/trending-web.png b/doc/screenshots/trending-web.png
index cd7e510b..a98ff47f 100644
Binary files a/doc/screenshots/trending-web.png and b/doc/screenshots/trending-web.png differ
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 3c4346f1..4715aaac 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -81,19 +81,13 @@ def event_stream():
def get_queues(r):
# We may want to put the llen in a pipeline to do only one query.
- data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()]
newData = []
-
- curr_range = 50
- for queue, card in data:
+ for queue, card in r.hgetall("queues").iteritems():
key = "MODULE_" + queue + "_"
- for i in range(1, 50):
- curr_num = r.get("MODULE_"+ queue + "_" + str(i))
- if curr_num is None:
- curr_range = i
- break
+ keySet = "MODULE_TYPE_" + queue
- for moduleNum in range(1, curr_range):
+ for moduleNum in r.smembers(keySet):
+
value = r.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")
diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js
index 1bdb29eb..e527aafa 100644
--- a/var/www/static/js/indexjavascript.js
+++ b/var/www/static/js/indexjavascript.js
@@ -207,7 +207,7 @@ function create_queue_table() {
table.appendChild(tableHead);
table.appendChild(tableBody);
var heading = new Array();
- heading[0] = "Queue Name"
+ heading[0] = "Queue Name.PID"
heading[1] = "Amount"
var tr = document.createElement('TR');
tableHead.appendChild(tr);
@@ -244,12 +244,8 @@ function create_queue_table() {
Tablediv.appendChild(table);
}
-$(document).ready(function () {
- if (typeof glob_tabvar == "undefined")
- location.reload();
- if (typeof glob_tabvar.row1 == "undefined")
- location.reload();
+function load_queues() {
var data = [];
var data2 = [];
var tmp_tab = [];
@@ -259,13 +255,17 @@ $(document).ready(function () {
var x = new Date();
for (i = 0; i < glob_tabvar.row1.length; i++){
- if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
- tmp_tab2.push(0);
- curves_labels2.push(glob_tabvar.row1[i][0]);
+ if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
+ if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+ tmp_tab2.push(0);
+ curves_labels2.push(glob_tabvar.row1[i][0].split(".")[0]);
+ }
}
else {
- tmp_tab.push(0);
- curves_labels.push(glob_tabvar.row1[i][0]);
+ if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+ tmp_tab.push(0);
+ curves_labels.push(glob_tabvar.row1[i][0].split(".")[0]);
+ }
}
}
tmp_tab.unshift(x);
@@ -324,19 +324,29 @@ $(document).ready(function () {
update_values();
if($('#button-toggle-queues').prop('checked')){
+ $("#queue-color-legend").show();
create_queue_table();
}
else{
$("#queueing").html('');
+ $("#queue-color-legend").hide();
}
+ queues_pushed = []
for (i = 0; i < (glob_tabvar.row1).length; i++){
- if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
- tmp_values2.push(glob_tabvar.row1[i][1]);
+ if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
+ if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+ queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+ tmp_values2.push(parseInt(glob_tabvar.row1[i][1]));
+ }
}
else {
- tmp_values.push(glob_tabvar.row1[i][1]);
+ if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+ queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+ tmp_values.push(parseInt(glob_tabvar.row1[i][1]));
+ }
+
}
}
tmp_values.unshift(x);
@@ -375,7 +385,19 @@ $(document).ready(function () {
// something went wrong, hide the canvas container
document.getElementById('myCanvasContainer').style.display = 'none';
}
+}
+function manage_undefined() {
+ if (typeof glob_tabvar == "undefined")
+ setTimeout(function() { if (typeof glob_tabvar == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+ else if (typeof glob_tabvar.row1 == "undefined")
+ setTimeout(function() { if (typeof glob_tabvar.row1 == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+ else
+ load_queues();
+}
+
+$(document).ready(function () {
+ manage_undefined();
});
diff --git a/var/www/templates/index.html b/var/www/templates/index.html
index 28c3aff9..8bbb59e1 100644
--- a/var/www/templates/index.html
+++ b/var/www/templates/index.html
@@ -66,7 +66,18 @@
Display queues
-
+
+
+
+
+
+ | Working queues |
+ | Idling queues |
+ | Stucked queues |
+
+
+
+
 }})