From 21f2c43572fa2a431fb3e10c6de46439a16e8aa4 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 25 Aug 2016 13:11:16 +0200 Subject: [PATCH 01/12] Individualized sentiments-graph loading --- var/www/Flask_server.py | 34 ++- var/www/static/js/sentiment_trending.js | 265 ++++-------------- .../sentiment_analysis_trending.html | 2 +- 3 files changed, 79 insertions(+), 222 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 8b6e05e3..1ab98d60 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -472,7 +472,7 @@ def sentiment_analysis_trending(): return render_template("sentiment_analysis_trending.html") -@app.route("/sentiment_analysis_getplotdata/") +@app.route("/sentiment_analysis_getplotdata/", methods=['GET']) def sentiment_analysis_getplotdata(): # Get the top providers based on number of pastes oneHour = 60*60 @@ -481,19 +481,22 @@ def sentiment_analysis_getplotdata(): dateStart = dateStart.replace(minute=0, second=0, microsecond=0) dateStart_timestamp = calendar.timegm(dateStart.timetuple()) - to_return = {} - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) - # if empty, get yesterday top providers - print 'providers_set_'+ get_date_range(1)[1] - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers - # if still empty, takes from all providers - if range_providers == []: - print 'today provider empty' - range_providers = r_serv_charts.smembers('all_provider_set') + getAllProviders = request.args.get('getProviders') + provider = request.args.get('provider') + if getAllProviders == 'True': + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) + # if empty, get yesterday top providers + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers + # if still empty, takes from all providers + if range_providers == []: + print 'today provider empty' + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(range_providers) - for cur_provider in range_providers: - print cur_provider - cur_provider_name = cur_provider + '_' + elif provider is not None: + to_return = {} + + cur_provider_name = provider + '_' list_date = {} for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): cur_set_name = cur_provider_name + str(cur_timestamp) @@ -503,9 +506,10 @@ def sentiment_analysis_getplotdata(): cur_value = r_serv_sentiment.get(cur_id) list_value.append(cur_value) list_date[cur_timestamp] = list_value - to_return[cur_provider] = list_date + to_return[provider] = list_date - return jsonify(to_return) + return jsonify(to_return) + return "Bad request" diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index a56723b8..af1614c3 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -1,4 +1,6 @@ +/* Functions and config */ + function generate_offset_to_time(num){ var to_ret = {}; for(i=0; i<=num; i++) { @@ -15,7 +17,7 @@ var to_ret = {}; for(i=day; i>=0; i--){ for(j=0; j<24; j++){ - var t1 =now.getDate()-i + ":"; + var t1 =now.getDate()-i + ":"; var t2 =now.getHours()-(23-j); t2 = t2 < 0 ? 24+t2 : t2; t2 += "h"; @@ -40,34 +42,52 @@ barColor: '#00bf5f', negBarColor: '#f22929', zeroColor: '#ffff00', - + tooltipFormat: '● {{offset:names}}, {{value}} ', }; -$.getJSON("/sentiment_analysis_getplotdata/", - function(data) { - var all_data = []; - var plot_data = []; - var graph_avg = []; - var array_provider = Object.keys(data); - var dates_providers = Object.keys(data[array_provider[0]]); - var dateStart = parseInt(dates_providers[0]); - var oneHour = 60*60; - var oneWeek = oneHour*24*7; +/* Plot and queries */ - var all_graph_day_sum = 0.0; - var all_graph_hour_sum = 0.0; - var all_graph_hour_maxVal = 0.0; - var all_day_avg = 0.0; - var all_day_avg_maxVal = 0.0; +var all_graph_day_sum = 0.0; +var all_graph_hour_sum = 0.0; +var all_graph_hour_maxVal = 0.0; +var all_day_avg = 0.0; +var all_day_avg_maxVal = 0.0; +var graph_avg = []; +var all_data = []; + +function draw_page() { + $.getJSON("/sentiment_analysis_getplotdata/?getProviders=True", + function(data) { + var promises = []; + for(i=0; i all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal; - var curr_avg = curr_sum / (curr_sum_elem); + var curr_avg = curr_sum / (curr_sum_elem); if(isNaN(curr_avg)) curr_avg = 0.0 - //var curr_avg = curr_sum / (oneWeek/oneHour); - //var curr_avg = curr_sum / (spark_data.length); graph_avg.push([curr_provider, curr_avg]); plot_data.push(spark_data); all_data.push(graph_data); - + sparklineOptions.chartRangeMax = max_value; sparklineOptions.chartRangeMin = -max_value; @@ -141,11 +158,11 @@ $.getJSON("/sentiment_analysis_getplotdata/", var num = graphNum + 1; var placeholder = '.sparkLineStatsWeek' + num; sparklineOptions.barWidth = 2; - $(placeholder).sparkline(plot_data[graphNum], sparklineOptions); + $(placeholder).sparkline(plot_data[0], sparklineOptions); $(placeholder+'t').text(curr_provider); - var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); + var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); $(placeholder+'s').text(curr_avg_text); - + sparklineOptions.barWidth = 18; sparklineOptions.tooltipFormat = '● Avg: {{value}} ' $(placeholder+'b').sparkline([curr_avg], sparklineOptions); @@ -169,8 +186,8 @@ $.getJSON("/sentiment_analysis_getplotdata/", // print today - var data_length = plot_data[graphNum].length; - var data_today = plot_data[graphNum].slice(data_length-24, data_length); + var data_length = plot_data[0].length; + var data_today = plot_data[0].slice(data_length-24, data_length); placeholder = '.sparkLineStatsToday' + num; sparklineOptions.barWidth = 14; @@ -199,188 +216,24 @@ $.getJSON("/sentiment_analysis_getplotdata/", $(avgName).addClass("panel-warning") } - }//for loop - - - - /* ---------------- Gauge ---------------- */ - var gaugeOptions = { - animateEasing: true, - - elementWidth: 200, - elementHeight: 125, - - arcFillStart: 10, - arcFillEnd: 12, - arcFillTotal: 20, - incTot: 1.0, - - arcBgColorLight: 200, - arcBgColorSat: 0, - arcStrokeFg: 20, - arcStrokeBg: 30, - - colorArcFg: '#FF3300', - animateSpeed: 1, - - }; - // Clone object - var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); - var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); - - - - gaugeOptions.appendTo = '#gauge_today_last_hour'; - gaugeOptions.dialLabel = 'Last hour'; - gaugeOptions.elementId = 'gauge1'; - var piePercent = (all_graph_hour_sum / 8) / all_graph_hour_maxVal; - gaugeOptions.inc = piePercent; - var gauge_today_last_hour = new FlexGauge(gaugeOptions); - - gaugeOptions2.appendTo = '#gauge_today_last_days'; - gaugeOptions2.dialLabel = 'Today'; - gaugeOptions2.elementId = 'gauge2'; - //piePercent = (all_graph_day_sum / (8*24)) / max_value; - piePercent = (all_day_avg / 8) / all_day_avg_maxVal; - gaugeOptions2.inc = piePercent; - var gauge_today_last_days = new FlexGauge(gaugeOptions2); - - gaugeOptions3.appendTo = '#gauge_week'; - gaugeOptions3.dialLabel = 'Week'; - gaugeOptions3.elementId = 'gauge3'; - - var graph_avg_sum = 0.0; - var temp_max_val = 0.0; - for (i=0; i temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val; - } - - piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val; - gaugeOptions3.inc = piePercent; - var gauge_today_last_days = new FlexGauge(gaugeOptions3); - - - /* --------- Sort providers -------- */ - - graph_avg.sort(function(a, b){return b[1]-a[1]}); - - for (i=1; i<6; i++){ - $('.worst'+i).text(graph_avg[7-(i-1)][0]); - $('.best'+i).text(graph_avg[i-1][0]); - } - - /* ----------- CanvasJS ------------ */ - - var comp_sum_day_pos = 0.0; - var comp_sum_day_neg = 0.0; - var comp_sum_hour_pos = 0.0; - var comp_sum_hour_neg = 0.0; - for(graphNum=0; graphNum<8; graphNum++){ - curr_graphData = all_data[graphNum]; - var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length); - for (i=1; i< gauge_data.length; i++){ - comp_sum_day_pos += gauge_data[i].compoundPos; - comp_sum_day_neg += gauge_data[i].compoundNeg; - - if(i == 23){ - comp_sum_hour_pos += gauge_data[i].compoundPos; - comp_sum_hour_neg += gauge_data[i].compoundNeg; - } - } } - - var options_canvasJS_1 = { - - animationEnabled: true, - axisY: { - tickThickness: 0, - lineThickness: 0, - valueFormatString: " ", - gridThickness: 0 - }, - axisX: { - tickThickness: 0, - lineThickness: 0, - labelFontSize: 0.1, - }, - data: [ - { - toolTipContent: "Positive: {y}", - type: "bar", - color: "green", - dataPoints: [ - {y: comp_sum_hour_pos/8} - ] - }, - { - toolTipContent: "Negative: {y}", - type: "bar", - color: "red", - dataPoints: [ - {y: comp_sum_hour_neg/8} - ] - } - ] - }; - - var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1); - - var options_canvasJS_2 = { - - animationEnabled: true, - axisY: { - tickThickness: 0, - lineThickness: 0, - valueFormatString: " ", - gridThickness: 0 - }, - axisX: { - tickThickness: 0, - lineThickness: 0, - labelFontSize: 0.1, - }, - data: [ - { - toolTipContent: "Positive: {y}", - type: "bar", - color: "green", - dataPoints: [ - {y: comp_sum_day_pos/8} - ] - }, - { - toolTipContent: "Negative: {y}", - type: "bar", - color: "red", - dataPoints: [ - {y: comp_sum_day_neg/8} - ] - } - ] - }; - - var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2); - - chart_canvas1.render(); - chart_canvas2.render(); - - - - } -); - - - - - - - - + ); + return query_plot +} +function draw_widgets() { + /* ---------------- Gauge ---------------- */ + var gaugeOptions = { + animateEasing: true, + elementWidth: 200, + elementHeight: 125, + arcFillStart: 10, + arcFillEnd: 12, + arcFillTotal: 20, + incTot: 1.0, diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 3a927251..944fab1d 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -508,7 +508,7 @@ $("#"+activePage).addClass("active"); $('[data-toggle="tooltip"]').tooltip(); - + draw_page(); // Reload every 30min setTimeout(function(){ location.reload(); }, 30*60*1000); }); From 0b5a3e0cd2587aea6c95e0e24052f2d75dc089d0 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 25 Aug 2016 15:06:18 +0200 Subject: [PATCH 02/12] Auto generate graph div + possibility to query all sentiments data. --- var/www/Flask_server.py | 19 +- var/www/static/js/sentiment_trending.js | 229 ++++++++++++++- .../sentiment_analysis_trending.html | 275 +----------------- 3 files changed, 244 insertions(+), 279 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 1ab98d60..48d4c3df 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -483,15 +483,20 @@ def sentiment_analysis_getplotdata(): getAllProviders = request.args.get('getProviders') provider = request.args.get('provider') + allProvider = request.args.get('all') if getAllProviders == 'True': - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) - # if empty, get yesterday top providers - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers - # if still empty, takes from all providers - if range_providers == []: - print 'today provider empty' + if allProvider == "True": range_providers = r_serv_charts.smembers('all_provider_set') - return jsonify(range_providers) + return jsonify(list(range_providers)) + else: + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) + # if empty, get yesterday top providers + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers + # if still empty, takes from all providers + if range_providers == []: + print 'today provider empty' + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(range_providers) elif provider is not None: to_return = {} diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index af1614c3..66fcac9e 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -1,5 +1,41 @@ /* Functions and config */ +function add_new_graph_today(id) { + return "
" + + "
" + + "Graph "+id+"" + + "Avg" + + "
" + + "
" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "
" + + "
" + + "
"; +}; +function add_new_graph_week(id) { + return "
" + + "
" + + "Graph "+id+"" + + "Avg" + + "
" + + "
" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "
" + + "
" + + "
"; +} function generate_offset_to_time(num){ var to_ret = {}; @@ -51,23 +87,45 @@ var all_graph_day_sum = 0.0; var all_graph_hour_sum = 0.0; +var all_graph_hour_sum_minus = 0.0; var all_graph_hour_maxVal = 0.0; var all_day_avg = 0.0; var all_day_avg_maxVal = 0.0; var graph_avg = []; var all_data = []; +var provider_already_loaded = []; +var totNumGraph = 0; + +function draw_page(all) { + $.getJSON("/sentiment_analysis_getplotdata/?getProviders=True&all="+all, + function(data) { + var promises = []; + + var the_length = provider_already_loaded.length == 0 ? 0 : provider_already_loaded.length; + for(i=0; i 0 ? 0 : 1; all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal; var curr_avg = curr_sum / (curr_sum_elem); @@ -225,6 +284,7 @@ function query_and_plot(provider, graphNum) { function draw_widgets() { + /* ---------------- Gauge ---------------- */ var gaugeOptions = { animateEasing: true, @@ -237,3 +297,154 @@ function draw_widgets() { arcFillTotal: 20, incTot: 1.0, + arcBgColorLight: 200, + arcBgColorSat: 0, + arcStrokeFg: 20, + arcStrokeBg: 30, + + colorArcFg: '#FF3300', + animateSpeed: 1, + + }; + // Clone object + var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); + var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); + + + + gaugeOptions.appendTo = '#gauge_today_last_hour'; + gaugeOptions.dialLabel = 'Last hour'; + gaugeOptions.elementId = 'gauge1'; + var piePercent = (all_graph_hour_sum / (totNumGraph - all_graph_hour_sum_minus)) / all_graph_hour_maxVal; + gaugeOptions.inc = piePercent; + var gauge_today_last_hour = new FlexGauge(gaugeOptions); + + gaugeOptions2.appendTo = '#gauge_today_last_days'; + gaugeOptions2.dialLabel = 'Today'; + gaugeOptions2.elementId = 'gauge2'; + piePercent = (all_day_avg / totNumGraph) / all_day_avg_maxVal; + gaugeOptions2.inc = piePercent; + var gauge_today_last_days = new FlexGauge(gaugeOptions2); + + gaugeOptions3.appendTo = '#gauge_week'; + gaugeOptions3.dialLabel = 'Week'; + gaugeOptions3.elementId = 'gauge3'; + + var graph_avg_sum = 0.0; + var temp_max_val = 0.0; + for (i=0; i temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val; + } + + piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val; + gaugeOptions3.inc = piePercent; + var gauge_today_last_days = new FlexGauge(gaugeOptions3); + + + /* --------- Sort providers -------- */ + + graph_avg.sort(function(a, b){return b[1]-a[1]}); + + for (i=1; i<6; i++){ + $('.worst'+i).text(graph_avg[7-(i-1)][0]); + $('.best'+i).text(graph_avg[i-1][0]); + } + + /* ----------- CanvasJS ------------ */ + + var comp_sum_day_pos = 0.0; + var comp_sum_day_neg = 0.0; + var comp_sum_hour_pos = 0.0; + var comp_sum_hour_neg = 0.0; + for(graphNum=0; graphNumPositive: {y}", + type: "bar", + color: "green", + dataPoints: [ + {y: comp_sum_hour_pos/totNumGraph} + ] + }, + { + toolTipContent: "Negative: {y}", + type: "bar", + color: "red", + dataPoints: [ + {y: comp_sum_hour_neg/totNumGraph} + ] + } + ] + }; + + var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1); + + var options_canvasJS_2 = { + + animationEnabled: true, + axisY: { + tickThickness: 0, + lineThickness: 0, + valueFormatString: " ", + gridThickness: 0 + }, + axisX: { + tickThickness: 0, + lineThickness: 0, + labelFontSize: 0.1, + }, + data: [ + { + toolTipContent: "Positive: {y}", + type: "bar", + color: "green", + dataPoints: [ + {y: comp_sum_day_pos/totNumGraph} + ] + }, + { + toolTipContent: "Negative: {y}", + type: "bar", + color: "red", + dataPoints: [ + {y: comp_sum_day_neg/totNumGraph} + ] + } + ] + }; + + var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2); + + chart_canvas1.render(); + chart_canvas2.render(); + +} diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 944fab1d..e1788f35 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -113,6 +113,7 @@
+
@@ -128,139 +129,13 @@
-
-
-
- Graph 1 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 2 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 3 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 4 - Avg -
-
- - - - - - - -
-
-
+
+
-
-
-
- Graph 5 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 6 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 7 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 8 - Avg -
-
- - - - - - - -
-
-
+
+
@@ -304,139 +179,13 @@
-
-
-
- Graph 1 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 2 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 3 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 4 - Avg -
-
- - - - - - - -
-
-
+
+
-
-
-
- Graph 5 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 6 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 7 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 8 - Avg -
-
- - - - - - - -
-
-
+
+
@@ -490,8 +239,6 @@
- -
@@ -502,13 +249,15 @@ - From aaac798db9ad3e17c5b5b96a34d57daa7ac1dfe3 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 25 Aug 2016 16:51:57 +0200 Subject: [PATCH 06/12] updated individual launch script --- bin/launch_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index 1cdde370..3c4fd548 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -56,4 +56,4 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py; read x' + screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x' From 813cc7031a99713d4ab64332cd9c77454f7e8326 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 25 Aug 2016 17:48:12 +0200 Subject: [PATCH 07/12] Changed lots of code into zrank --- bin/WebStats.py | 20 +------------------- var/www/Flask_server.py | 17 ++++------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/bin/WebStats.py b/bin/WebStats.py index 20c1f489..d8ff0876 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -66,25 +66,7 @@ def compute_progression(server, field_name, num_day, url_parsed): # filter if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase): - if server.sismember(redis_progression_name_set, keyword): #if keyword is in the set - server.hset(redis_progression_name, keyword, keyword_increase) #update its value - - elif (server.scard(redis_progression_name_set) < max_set_cardinality): - server.sadd(redis_progression_name_set, keyword) - - else: #not in the set - #Check value for all members - member_set = [] - for keyw in server.smembers(redis_progression_name_set): - member_set.append((keyw, int(server.hget(redis_progression_name, keyw)))) - member_set.sort(key=lambda tup: tup[1]) - if member_set[0][1] < keyword_increase: - print 'removing', member_set[0][0] + '('+str(member_set[0][1])+')', 'and adding', keyword, str(keyword_increase) - #remove min from set and add the new one - server.srem(redis_progression_name_set, member_set[0][0]) - server.sadd(redis_progression_name_set, keyword) - server.hdel(redis_progression_name, member_set[0][0]) - server.hset(redis_progression_name, keyword, keyword_increase) + server.zadd("z_top_progression_"+field_name, float(keyword_increase), keyword) if __name__ == '__main__': diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 14bb8474..925ed369 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -160,6 +160,7 @@ def showpaste(content_range): p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') p_duplicate_full_list = parseStringToList2(paste._get_p_duplicate()) + p_duplicate_full_list = json.load(paste._get_p_duplicate()) p_duplicate_list = [] p_simil_list = [] p_hashtype_list = [] @@ -290,19 +291,9 @@ def progressionCharts(): return jsonify(bar_values) else: - redis_progression_name = 'top_progression_'+trending_name - redis_progression_name_set = 'top_progression_'+trending_name+'_set' - - # Iterate over element in top_x_set and retreive their value - member_set = [] - for keyw in r_serv_charts.smembers(redis_progression_name_set): - keyw_value = r_serv_charts.hget(redis_progression_name, keyw) - keyw_value = keyw_value if keyw_value is not None else 0 - member_set.append((keyw, int(keyw_value))) - member_set.sort(key=lambda tup: tup[1], reverse=True) - if len(member_set) == 0: - member_set.append(("No relevant data", int(100))) - return jsonify(member_set) + redis_progression_name = "z_top_progression_" + trending_name + keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10) + return jsonify(keyw_value) @app.route("/_moduleCharts", methods=['GET']) def modulesCharts(): From 07d755906062cc223d405bb0008f8ff4021adab7 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 08:30:09 +0200 Subject: [PATCH 08/12] Added dataTable support in browse paste modal. --- var/www/templates/terms_management.html | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 3093837f..32a51c32 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -233,15 +233,15 @@ $.getJSON(url, function (data) { if (data.length != 0) { var html_to_add = ""; - html_to_add += ""; + html_to_add += "
"; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; - html_to_add += ""; - html_to_add += ""; + html_to_add += ""; + html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; @@ -253,8 +253,8 @@ html_to_add += ""; html_to_add += ""; html_to_add += ""; - html_to_add += ""; - html_to_add += ""; + html_to_add += ""; + html_to_add += ""; html_to_add += ""; html_to_add += ""; @@ -264,6 +264,7 @@ $("#mymodalbody").html(html_to_add); $("[data-toggle=popover]").popover(); $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); + $('#modal-table').DataTable(); } else { $("#mymodalbody").html("No paste containing this term has been received yet."); $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); From 8b33b4e7bad0475cf4ce080cfc3b7a8b2336ea41 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 09:21:02 +0200 Subject: [PATCH 09/12] Added not running module info + swapped order of module launch --- bin/LAUNCH.sh | 4 +- bin/ModuleInformation.py | 154 +++++++++++++++++++++---------------- bin/launch_scripts.sh | 4 +- doc/generate_graph_data.py | 3 + 4 files changed, 93 insertions(+), 72 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 024b22e4..b50a75d6 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -112,6 +112,8 @@ function launching_scripts { echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' + sleep 0.1 screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' @@ -159,8 +161,6 @@ function launching_scripts { screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x' } diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py index a3c0f0c2..16a0e08f 100755 --- a/bin/ModuleInformation.py +++ b/bin/ModuleInformation.py @@ -87,82 +87,100 @@ if __name__ == "__main__": if args.clear == 1: clearRedisModuleInfo() - while True: - curr_range = 50 - printarray1 = [] - printarray2 = [] - for queue, card in server.hgetall("queues").iteritems(): - key = "MODULE_" + queue + "_" - for i in range(1, 50): - curr_num = server.get("MODULE_"+ queue + "_" + str(i)) - if curr_num is None: - curr_range = i - break + module_file_array = set() + with open('../doc/all_modules.txt', 'r') as module_file: + for line in module_file: + module_file_array.add(line[:-1]) - for moduleNum in range(1, curr_range): - value = server.get(key + str(moduleNum)) - if value is not None: - timestamp, path = value.split(", ") - if timestamp is not None and path is not None: - startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) - processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] + while True: - if int(card) > 0: - if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module: - log = open(log_filename, 'a') - log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") - if args.autokill == 1: - kill_module(queue) + all_queue = set() + curr_range = 50 + printarray1 = [] + printarray2 = [] + printarray3 = [] + for queue, card in server.hgetall("queues").iteritems(): + all_queue.add(queue) + key = "MODULE_" + queue + "_" + for i in range(1, 50): + curr_num = server.get("MODULE_"+ queue + "_" + str(i)) + if curr_num is None: + curr_range = i + break - printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) + for moduleNum in range(1, curr_range): + value = server.get(key + str(moduleNum)) + if value is not None: + timestamp, path = value.split(", ") + if timestamp is not None and path is not None: + startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) + processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] - else: - printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) + if int(card) > 0: + if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module: + log = open(log_filename, 'a') + log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") + if args.autokill == 1: + kill_module(queue) - printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) - printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) - printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) - printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) + printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) - os.system('clear') - t1 = AsciiTable(printarray1, title="Working queues") - t1.column_max_width(1) - if not t1.ok: - longest_col = t1.column_widths.index(max(t1.column_widths)) - max_length_col = t1.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t1.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t1.table_data[i] = content + else: + printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) - t2 = AsciiTable(printarray2, title="Idling queues") - t2.column_max_width(1) - if not t2.ok: - longest_col = t2.column_widths.index(max(t2.column_widths)) - max_length_col = t2.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t2.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t2.table_data[i] = content + for curr_queue in module_file_array: + if curr_queue not in all_queue: + printarray3.append([curr_queue, "Not running"]) + printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) + printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) + printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) + printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) + printarray3.insert(0,["Queue", "State"]) - print t1.table - print '\n' - print t2.table + os.system('clear') + t1 = AsciiTable(printarray1, title="Working queues") + t1.column_max_width(1) + if not t1.ok: + longest_col = t1.column_widths.index(max(t1.column_widths)) + max_length_col = t1.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t1.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t1.table_data[i] = content - time.sleep(args.refresh) + t2 = AsciiTable(printarray2, title="Idling queues") + t2.column_max_width(1) + if not t2.ok: + longest_col = t2.column_widths.index(max(t2.column_widths)) + max_length_col = t2.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t2.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t2.table_data[i] = content + + t3 = AsciiTable(printarray3, title="Not running queues") + t3.column_max_width(1) + + print t1.table + print '\n' + print t2.table + print '\n' + print t3.table + + time.sleep(args.refresh) diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index 3c4fd548..694f0138 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -8,6 +8,8 @@ sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' + sleep 0.1 screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' @@ -55,5 +57,3 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x' diff --git a/doc/generate_graph_data.py b/doc/generate_graph_data.py index cff1f538..c1b4b98e 100755 --- a/doc/generate_graph_data.py +++ b/doc/generate_graph_data.py @@ -32,6 +32,9 @@ with open('../bin/packages/modules.cfg', 'r') as f: continue output_set_graph = set() + with open('all_modules.txt', 'w') as f2: + for e in all_modules: + f2.write(e+"\n") for module in modules.keys(): for stream_in in modules[module]['sub']: From 86b655100e906e9aafe8eb563e1ef960c2da1795 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 10:01:14 +0200 Subject: [PATCH 10/12] Deleted useless parsing function --- var/www/Flask_server.py | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 925ed369..3c4346f1 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -112,40 +112,6 @@ def list_len(s): return len(s) app.jinja_env.filters['list_len'] = list_len -def parseStringToList(the_string): - strList = "" - elemList = [] - for c in the_string: - if c != ']': - if c != '[' and c !=' ' and c != '"': - strList += c - else: - the_list = strList.split(',') - if len(the_list) == 3: - elemList = elemList + the_list - elif len(the_list) == 2: - elemList.append(the_list) - elif len(the_list) > 1: - elemList.append(the_list[1:]) - strList = "" - return elemList - -def parseStringToList2(the_string): - if the_string == []: - return [] - else: - res = [] - tab_str = the_string.split('], [') - tab_str[0] = tab_str[0][1:]+']' - tab_str[len(tab_str)-1] = '['+tab_str[len(tab_str)-1][:-1] - res.append(parseStringToList(tab_str[0])) - for i in range(1, len(tab_str)-2): - tab_str[i] = '['+tab_str[i]+']' - res.append(parseStringToList(tab_str[i])) - if len(tab_str) > 1: - res.append(parseStringToList(tab_str[len(tab_str)-1])) - return res - def showpaste(content_range): requested_path = request.args.get('paste', '') @@ -159,8 +125,7 @@ def showpaste(content_range): p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') - p_duplicate_full_list = parseStringToList2(paste._get_p_duplicate()) - p_duplicate_full_list = json.load(paste._get_p_duplicate()) + p_duplicate_full_list = json.loads(paste._get_p_duplicate()) p_duplicate_list = [] p_simil_list = [] p_hashtype_list = [] @@ -184,7 +149,7 @@ def showpaste(content_range): hash_types = [] comp_vals = [] for i in indices: - hash_types.append(p_duplicate_full_list[i][0]) + hash_types.append(p_duplicate_full_list[i][0].encode('utf8')) comp_vals.append(p_duplicate_full_list[i][2]) dup_list_removed.append(i) From 494980c48e3fd277655850ee2c5b3f79ba45a624 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 10:07:06 +0200 Subject: [PATCH 11/12] Added comment in moduleInformation --- bin/ModuleInformation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py index 16a0e08f..d783418f 100755 --- a/bin/ModuleInformation.py +++ b/bin/ModuleInformation.py @@ -1,6 +1,18 @@ #!/usr/bin/env python2 # -*-coding:UTF-8 -* +''' + +This module can be use to see information of running modules. +These information are logged in "logs/moduleInfo.log" + +It can also try to manage them by killing inactive one. +However, it does not support mutliple occurence of the same module +(It will kill the first one obtained by get) + + +''' + import time import datetime import redis From 4cb3b672bedcc9e5367edd2ce275fe615b5b5b35 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 10:28:35 +0200 Subject: [PATCH 12/12] Added comments --- var/www/static/js/indexjavascript.js | 6 ++++++ var/www/static/js/sentiment_trending.js | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index d1850d67..1bdb29eb 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -227,6 +227,12 @@ function create_queue_table() { td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum)); tr.appendChild(td) } + // Used to decide the color of the row + // We have glob_tabvar.row1[][j] with: + // - j=0: ModuleName + // - j=1: queueLength + // - j=2: LastProcessedPasteTime + // - j=3: Number of the module belonging in the same category if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2) tr.className += " danger"; else if (parseInt(glob_tabvar.row1[i][2]) > 60*1) diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index 66fcac9e..d96c6fbb 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -96,6 +96,9 @@ var all_data = []; var provider_already_loaded = []; var totNumGraph = 0; +// Query all providers name then launch the query and plot process for each of them. +// When everything is terminated, plot the widgets (Gauge, canvasJS, table) +// input: all - set to 'True' if you take all providers function draw_page(all) { $.getJSON("/sentiment_analysis_getplotdata/?getProviders=True&all="+all, function(data) { @@ -130,7 +133,9 @@ function draw_page(all) { } - +// Query data and plot it for a given provider +// input - provider: The povider name to be plotted +// input - graphNum: The number of the graph (Use to plot on correct div) function query_and_plot(provider, graphNum) { var query_plot = $.getJSON("/sentiment_analysis_getplotdata/?provider="+provider, function(data) { @@ -186,8 +191,7 @@ function query_and_plot(provider, graphNum) { max_value = Math.abs(pos-neg) > max_value ? Math.abs(pos-neg) : max_value; if(curr_date >= dateStart+oneWeek-23*oneHour){ - max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day; - day_sum += (pos-neg); + max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day; day_sum += (pos-neg); day_sum_elem++; } if(curr_date > dateStart+oneWeek-2*oneHour && curr_date <=dateStart+oneWeek-oneHour){
SourceDateEncodingSize (Kb)Mime(# lines, Max line length)# linesMax lengthPreview
"+curr_data.date+""+curr_data.encoding+""+curr_data.size+""+curr_data.mime+"("+curr_data.lineinfo[0]+", "+curr_data.lineinfo[1]+")"+curr_data.lineinfo[0]+""+curr_data.lineinfo[1]+"