From ab61e32399075580963e3942969b68f5bb949a2e Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 13 Jul 2016 08:59:48 +0200 Subject: [PATCH 01/16] Commented out get_language because it adds too much overhead --- bin/Attribute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/Attribute.py b/bin/Attribute.py index 46d80858..a7f78696 100755 --- a/bin/Attribute.py +++ b/bin/Attribute.py @@ -51,12 +51,13 @@ if __name__ == "__main__": PST = Paste.Paste(message) else: publisher.debug("Script Attribute is idling 1s") + print 'sleeping' time.sleep(1) continue # FIXME do it directly in the class PST.save_attribute_redis("p_encoding", PST._get_p_encoding()) - PST.save_attribute_redis("p_language", PST._get_p_language()) + #PST.save_attribute_redis("p_language", PST._get_p_language()) # FIXME why not all saving everything there. PST.save_all_attributes_redis() # FIXME Not used. From fba14bfb4b14ed3017d3e07508fa595d47bf5278 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 13 Jul 2016 15:57:33 +0200 Subject: [PATCH 02/16] In index: Added number of processed pastes chart --- bin/Global.py | 9 +++++ var/www/Flask_server.py | 3 +- var/www/static/js/indexjavascript.js | 60 ++++++++++++++++++++++++++++ var/www/templates/index.html | 15 +++++-- 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/bin/Global.py b/bin/Global.py index fb44c70b..8b6e482f 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -31,6 +31,8 @@ from Helper import Process if __name__ == '__main__': publisher.port = 6380 publisher.channel = 'Script' + processed_paste = 0 + time_1 = time.time() config_section = 'Global' @@ -54,6 +56,12 @@ if __name__ == '__main__': continue else: print "Empty Queues: Waiting..." + if int(time.time() - time_1) > 30: + to_print = 'Global; ; ; ;glob Processed {0} paste(s)'.format(processed_paste) + print to_print + publisher.info(to_print) + time_1 = time.time() + processed_paste = 0 time.sleep(1) continue # Creating the full filepath @@ -66,3 +74,4 @@ if __name__ == '__main__': with open(filename, 'wb') as f: f.write(base64.standard_b64decode(gzip64encoded)) p.populate_set_out(filename) + processed_paste+=1 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 018608f1..36fcfbcc 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -122,7 +122,8 @@ def search(): @app.route("/") def index(): - return render_template("index.html") + default_minute = cfg.get("Flask", "minute_processed_paste") + return render_template("index.html", default_minute = default_minute) @app.route("/monitoring/") diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 7eb7e5c5..ef9bf0c6 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -1,3 +1,54 @@ +// Plot and update the number of processed pastes +$(function() { + var data = []; + var totalPoints = 60*10; //60s*10m + var curr_max = 0; + + function getData() { + if (data.length > 0){ + curr_max = curr_max == data[0] ? Math.max.apply(null, data) : curr_max; + data = data.slice(1); + } + + while (data.length < totalPoints) { + var y = (typeof window.paste_num_tabvar !== "undefined") ? window.paste_num_tabvar : 0; + curr_max = curr_max < y ? y : curr_max; + data.push(y); + } + + // Zip the generated y values with the x values + var res = []; + for (var i = 0; i < data.length; ++i) { + res.push([i, data[i]]) + } + return res; + } + + var updateInterval = 1000; + var options = { + series: { shadowSize: 1 }, + lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}, + yaxis: { min: 0, max: 40 }, + xaxis: { show: false }, + colors: ["#F4A506"], + grid: { + tickColor: "#dddddd", + borderWidth: 0 + }, + }; + var plot = $.plot("#realtimechart", [ getData() ], options); + + function update() { +console.log(curr_max); + plot.setData([getData()]); + plot.getOptions().yaxes[0].max = curr_max; + plot.setupGrid(); + plot.draw(); + setTimeout(update, updateInterval); + } + update(); +}); + function initfunc( csvay, scroot) { window.csv = csvay; window.scroot = scroot; @@ -38,6 +89,13 @@ function create_log_table(obj_json) { var chansplit = obj_json.channel.split('.'); var parsedmess = obj_json.data.split(';'); + if (parsedmess[0] == "Global"){ + var paste_processed = parsedmess[4].split(" ")[2]; + console.log(paste_processed) + window.paste_num_tabvar = paste_processed; + return; + } + if( chansplit[1] == "INFO" ){ tr.className = "info"; } @@ -270,3 +328,5 @@ $(document).ready(function () { } }); + + diff --git a/var/www/templates/index.html b/var/www/templates/index.html index 70d314b0..dc81cfe3 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -14,9 +14,10 @@ - + + - From 321064bf0d4b905949db9d9571414971c74579d6 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 14 Jul 2016 10:31:47 +0200 Subject: [PATCH 03/16] Fixed a bug in processed_pastes graph --- var/www/static/js/indexjavascript.js | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index ef9bf0c6..628ffe86 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -6,16 +6,16 @@ $(function() { function getData() { if (data.length > 0){ - curr_max = curr_max == data[0] ? Math.max.apply(null, data) : curr_max; - data = data.slice(1); + var data_old = data[0]; + data = data.slice(1); + curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max; } while (data.length < totalPoints) { - var y = (typeof window.paste_num_tabvar !== "undefined") ? window.paste_num_tabvar : 0; - curr_max = curr_max < y ? y : curr_max; - data.push(y); + var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0; + curr_max = y > curr_max ? y : curr_max; + data.push(y); } - // Zip the generated y values with the x values var res = []; for (var i = 0; i < data.length; ++i) { @@ -29,17 +29,15 @@ $(function() { series: { shadowSize: 1 }, lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}, yaxis: { min: 0, max: 40 }, - xaxis: { show: false }, - colors: ["#F4A506"], + colors: ["#a971ff"], grid: { - tickColor: "#dddddd", - borderWidth: 0 + tickColor: "#dddddd", + borderWidth: 0 }, }; var plot = $.plot("#realtimechart", [ getData() ], options); function update() { -console.log(curr_max); plot.setData([getData()]); plot.getOptions().yaxes[0].max = curr_max; plot.setupGrid(); @@ -91,7 +89,6 @@ function create_log_table(obj_json) { if (parsedmess[0] == "Global"){ var paste_processed = parsedmess[4].split(" ")[2]; - console.log(paste_processed) window.paste_num_tabvar = paste_processed; return; } From 9a9e07f600cfdf769ecb72934b2b418569d3c8e2 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 15 Jul 2016 09:08:38 +0200 Subject: [PATCH 04/16] Added default configuration --- bin/packages/config.cfg.sample | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6d07707c..853edd2a 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -1,9 +1,27 @@ [Directories] bloomfilters = Blooms pastes = PASTES + wordtrending_csv = var/www/static/csv/wordstrendingdata wordsfile = files/wordfile +protocolstrending_csv = var/www/static/csv/protocolstrendingdata +protocolsfile = files/protocolsfile + +tldstrending_csv = var/www/static/csv/tldstrendingdata +tldsfile = AILENV/lib/faup/src/data/mozilla.tlds + +domainstrending_csv = var/www/static/csv/domainstrendingdata + +##### Flask ##### +[Flask] +#Maximum number of character to display in the toolip +max_preview_char = 250 +#Maximum number of character to display in the modal +max_preview_modal = 800 +#Default number of header to display in trending graphs +default_display = 10 + ##### Redis ##### [Redis_Cache] host = localhost @@ -31,6 +49,11 @@ host = localhost port = 2013 db = 0 +[Redis_Level_DB_Domain] +host = localhost +port = 2016 +db = 3 + [Redis_Level_DB_Hashs] host = localhost port = 2013 From 2383db022f360ed0883572de975941146b95b0dd Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 15 Jul 2016 09:10:44 +0200 Subject: [PATCH 05/16] Added default configuration --- bin/packages/config.cfg.sample | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6d07707c..0bc6fe4b 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -4,6 +4,11 @@ pastes = PASTES wordtrending_csv = var/www/static/csv/wordstrendingdata wordsfile = files/wordfile +##### Flask ##### +[Flask] +#Number of minutes displayed for the number of processed pastes. +minute_processed_paste = 10 + ##### Redis ##### [Redis_Cache] host = localhost From 4ecb9a4b443472e6247f31092b08fe411a976ce2 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 15 Jul 2016 09:47:00 +0200 Subject: [PATCH 06/16] Fixed a flexibility issue --- var/www/static/js/indexjavascript.js | 5 +++-- var/www/templates/index.html | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 628ffe86..cef3bf41 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -1,7 +1,8 @@ // Plot and update the number of processed pastes $(function() { var data = []; - var totalPoints = 60*10; //60s*10m + var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10; + var totalPoints = 60*parseInt(default_minute); //60s*minute var curr_max = 0; function getData() { @@ -24,7 +25,7 @@ $(function() { return res; } - var updateInterval = 1000; + var updateInterval = 1000; //1s var options = { series: { shadowSize: 1 }, lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}, diff --git a/var/www/templates/index.html b/var/www/templates/index.html index dc81cfe3..99c84941 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -19,6 +19,7 @@ - WordsTrend diff --git a/var/www/templates/Trending.html b/var/www/templates/Trending.html index 8928f948..f99a88f9 100644 --- a/var/www/templates/Trending.html +++ b/var/www/templates/Trending.html @@ -14,9 +14,8 @@ - - @@ -56,7 +55,7 @@
- +
- -
+ +
{% include 'trending_graphs/Tldstrending.html' %}
@@ -75,7 +74,7 @@
{% include 'trending_graphs/Wordstrending.html' %} -
+
{% include 'trending_graphs/Protocolstrending.html' %}
@@ -98,27 +97,27 @@ current_displayed_graph = new Graph(pannel, path, header_size); setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300); }, 'text'); - + } // When a pannel is shown, create_and_plot. $('.nav-tabs a').on('shown.bs.tab', function(event){ - create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path')); + create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path')); });
diff --git a/var/www/templates/index.html b/var/www/templates/index.html index 0110f22e..f4559450 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -14,8 +14,6 @@ - - - -