From cd74e6d1c5b4a3330820d5f28cb6aa8a5021f2e6 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 11 Aug 2016 09:40:42 +0200 Subject: [PATCH 01/68] Added module sentimentAnalyser --- bin/SentimentAnalyser.py | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 bin/SentimentAnalyser.py diff --git a/bin/SentimentAnalyser.py b/bin/SentimentAnalyser.py new file mode 100644 index 00000000..efb2ede6 --- /dev/null +++ b/bin/SentimentAnalyser.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules + +nltk.sentiment.vader module: + Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text. Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014. + +""" + +import time +from pubsublogger import publisher +from Helper import Process + +from nltk.sentiment.vader import SentimentIntensityAnalyzer +from nltk import tokenize + + +def Analyse(message): + path = message + paste = Paste.paste(path) + content = paste.p_get_content() + + sentences = tokenize.sent_tokenize(content.decode('utf-8', 'ignore')) + + sid = SentimentIntensityAnalyzer() + for sentence in sentences: + print(sentence) + ss = sid.polarity_scores(sentence) + for k in sorted(ss): + print('{0}: {1}, '.format(k, ss[k])) + print '' + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = '
' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + Analyse(message) From 5612a77af9fd9d4f022578967ed0004e55bbd49b Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 11 Aug 2016 10:55:51 +0200 Subject: [PATCH 02/68] Separated webpage header code from all webpage --- var/www/templates/Moduletrending.html | 12 ++++-------- var/www/templates/Trending.html | 14 +++++--------- var/www/templates/browse_important_paste.html | 14 +++++--------- var/www/templates/header.html | 8 ++++++++ var/www/templates/index.html | 16 +++++++--------- 5 files changed, 29 insertions(+), 35 deletions(-) create mode 100644 var/www/templates/header.html diff --git a/var/www/templates/Moduletrending.html b/var/www/templates/Moduletrending.html index 0e51f95e..a2635868 100644 --- a/var/www/templates/Moduletrending.html +++ b/var/www/templates/Moduletrending.html @@ -24,14 +24,7 @@
- +
@@ -52,8 +52,10 @@ -
+
+ + - - - - diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index a54807b3..4db4d84d 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -11,16 +11,56 @@ - - - + + + + -
+
- +
@@ -50,10 +90,74 @@
+ +
+
+
+
+
+
+ Today's mood +
+
+ +
+ +
+
+
    +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    • +
  • Graph 5
    • +
+
+
+
+
+
    +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    • +
  • Graph 5
    • +
+
+
+
+ +
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+ +
+
+ + +
+
+ + + -
+
+
+ - - - - From c028ca80da06d8be4333baedf953f15753c3063e Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 11 Aug 2016 17:57:23 +0200 Subject: [PATCH 05/68] Separated options from the plot funcion for sparkline. --- bin/ModuleStats.py | 2 +- .../sentiment_analysis_trending.html | 175 ++++++++++++++---- 2 files changed, 140 insertions(+), 37 deletions(-) diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index 32c631ed..d5849951 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -15,7 +15,7 @@ from Helper import Process from packages import Paste # Config Var -max_set_cardinality = 7 +max_set_cardinality = 8 def get_date_range(num_day): curr_date = datetime.date.today() diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 4db4d84d..3d34101e 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -33,7 +33,7 @@ .sparkLineStats ul li { margin-bottom: 8px; - line-height: 60px; + line-height: 90px; padding-top: 3px; font-size: 12px; } @@ -53,6 +53,7 @@ position: relative; text-align: center; top: -60px; + margin-bottom: -10px; width: 100%; } @@ -106,22 +107,20 @@
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • -
  • Graph 5
    • +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • -
  • Graph 5
    • +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
@@ -146,8 +145,87 @@ - +
+
+
+
+
+
+ Week's mood +
+
+ +
+ +
+
+
    +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    • +
+
+
+
+
+
    +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    • +
+
+
+
+ +
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Worst moodBest mood
PastebinIdeone
PastebinIdeone
PastebinIdeone
PastebinIdeone
PastebinIdeone
+
+
+ +
+ +
+
+ +
+ +
@@ -171,7 +249,7 @@ animateEasing: true, elementWidth: 200, - elementHeight: 150, + elementHeight: 125, arcFillStart: 10, arcFillEnd: 12, @@ -187,39 +265,35 @@ }; var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); + var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); gaugeOptions.appendTo = '#gauge_today_last_hour'; gaugeOptions.dialLabel = 'Last hour'; gaugeOptions.elementId = 'gauge1'; - console.log(gaugeOptions); var gauge_today_last_hour = new FlexGauge(gaugeOptions); gaugeOptions2.appendTo = '#gauge_today_last_days'; gaugeOptions2.dialLabel = 'Today'; gaugeOptions2.elementId = 'gauge2'; - console.log(gaugeOptions2); var gauge_today_last_days = new FlexGauge(gaugeOptions2); + gaugeOptions3.appendTo = '#gauge_week'; + gaugeOptions3.dialLabel = 'Week'; + gaugeOptions3.elementId = 'gauge3'; + var gauge_today_last_days = new FlexGauge(gaugeOptions3); + + + + + + + + /* ---------- Sparkline Charts ---------- */ - //generate random number for charts - randNum = function(){ - //return Math.floor(Math.random()*101); - return (Math.floor( Math.random()* (1+40-20) ) ) + 20; - } - - var chartColours = ['#2FABE9', '#FA5833', '#b9e672', '#bbdce3', '#9a3b1b', '#5a8022', '#2c7282']; - - //sparklines (making loop with random data for all 7 sparkline) - i=1; - for (i=1; i<9; i++) { - var data = [[1, 3+randNum()], [2, 5+randNum()], [3, 8+randNum()], [4, 11+randNum()],[5, 14+randNum()],[6, 17+randNum()],[7, 20+randNum()], [8, 15+randNum()], [9, 18+randNum()], [10, 22+randNum()]]; - placeholder = '.sparkLineStats' + i; - - - $(placeholder).sparkline(data, { - width: 200,//Width of the chart - Defaults to 'auto' - May be any valid css width - 1.5em, 20px, etc (using a number without a unit specifier won't do what you want) - This option does nothing for bar and tristate chars (see barWidth) - height: 60,//Height of the chart - Defaults to 'auto' (line height of the containing tag) + var sparklineOptions = { + width: 250,//Width of the chart - Defaults to 'auto' - May be any valid css width - 1.5em, 20px, etc (using a number without a unit specifier won't do what you want) - This option does nothing for bar and tristate chars (see barWidth) + height: 80,//Height of the chart - Defaults to 'auto' (line height of the containing tag) lineColor: '#2FABE9',//Used by line and discrete charts to specify the colour of the line drawn as a CSS values string fillColor: '#f2f7f9',//Specify the colour used to fill the area under the graph as a CSS value. Set to false to disable fill spotColor: '#467e8c',//The CSS colour of the final value marker. Set to false or an empty string to hide it @@ -227,11 +301,40 @@ minSpotColor: '#FA5833',//The CSS colour of the marker displayed for the mimum value. Set to false or an empty string to hide it spotRadius: 2,//Radius of all spot markers, In pixels (default: 1.5) - Integer lineWidth: 1//In pixels (default: 1) - Integer - }); - -} + }; + //generate random number for charts + randNum = function(){ + var num = Math.random(); + if(num > 0.5) + num = -1+num; + console.log(Math.floor(num*101)); + return Math.floor(num*101); + //return (Math.floor( Math.random()* (1+40-20) ) ) + 20; + } + + var chartColours = ['#2FABE9', '#FA5833', '#b9e672', '#bbdce3', '#9a3b1b', '#5a8022', '#2c7282']; + + //sparklines (making loop with random data for all 10 sparkline) + i=1; + for (i=1; i<10; i++) { + var data = [[1, 3+randNum()], [2, 5+randNum()], [3, 8+randNum()], [4, 11+randNum()],[5, 14+randNum()],[6, 17+randNum()],[7, 20+randNum()], [8, 15+randNum()], [9, 18+randNum()], [10, 22+randNum()]]; + placeholder = '.sparkLineStatsToday' + i; + + $(placeholder).sparkline(data, sparklineOptions); + + } + + //sparklines (making loop with random data for all 10 sparkline) + i=1; + for (i=1; i<10; i++) { + var data = [[1, 3+randNum()], [2, 5+randNum()], [3, 8+randNum()], [4, 11+randNum()],[5, 14+randNum()],[6, 17+randNum()],[7, 20+randNum()], [8, 15+randNum()], [9, 18+randNum()], [10, 22+randNum()]]; + placeholder = '.sparkLineStatsWeek' + i; + + $(placeholder).sparkline(data, sparklineOptions); + + } From 87a86b38d1e99dc9b1c6c3a5f745cd10ac8dd9eb Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 11 Aug 2016 21:08:43 +0200 Subject: [PATCH 06/68] Added page Plot-tool --- .../sentiment_analysis_plot_tool.html | 132 ++++++++++++++++++ .../sentiment_analysis_trending.html | 2 +- 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/var/www/templates/sentiment_analysis_plot_tool.html b/var/www/templates/sentiment_analysis_plot_tool.html index ed4726bd..80861dc2 100644 --- a/var/www/templates/sentiment_analysis_plot_tool.html +++ b/var/www/templates/sentiment_analysis_plot_tool.html @@ -12,11 +12,20 @@ + + + + @@ -50,7 +59,83 @@
+ +
+
+
+
+
+
+ Select options +
+
+
+ +
+ +
+
+
    +
    • +
    • +
    • +
    • +
+
+
+
+
+
    +
    • +
    • +
    • +
    • +
+
+
+
+ +
+
+ Date:10/09/15 - 15/09/16 +
+
+ +
+ +
+
+ +
+
+ +
+ +
+
+ +
+
+
+
+
+
+ Graph +
+
+
+
+ +
+
+ +
+ +
+
+ +
@@ -63,6 +148,53 @@ activePage = $('h1.page-header').attr('data-page'); $("#"+activePage).addClass("active"); + $( ".sliderRange" ).slider({ + range: true, + min: 0, + max: 500, + values: [ 192, 470 ], + slide: function( event, ui ) { + $( ".sliderRangeLabel" ).html( "$" + ui.values[ 0 ] + " - $" + ui.values[ 1 ] ); + } + }); + + function randNum(){ + return ((Math.floor( Math.random()* (1+40-20) ) ) + 20)* 1200; + } + + + + +var likes = [[1, 5+randNum()], [2, 10+randNum()], [3, 15+randNum()], [4, 20+randNum()],[5, 25+randNum()],[6, 30+randNum()],[7, 35+randNum()],[8, 40+randNum()],[9, 45+randNum()],[10, 50+randNum()],[11, 55+randNum()],[12, 60+randNum()],[13, 65+randNum()],[14, 70+randNum()],[15, 75+randNum()],[16, 80+randNum()],[17, 85+randNum()],[18, 90+randNum()],[19, 85+randNum()],[20, 80+randNum()],[21, 75+randNum()],[22, 80+randNum()],[23, 75+randNum()],[24, 70+randNum()],[25, 65+randNum()],[26, 75+randNum()],[27,80+randNum()],[28, 85+randNum()],[29, 90+randNum()], [30, 95+randNum()]]; + + var plot = $.plot($("#graph"), + [ { data: likes, label: "Fans"} ], { + series: { + lines: { show: true, + lineWidth: 2, + fill: true, fillColor: { colors: [ { opacity: 0.5 }, { opacity: 0.2 } ] } + }, + points: { show: true, + lineWidth: 2 + }, + shadowSize: 0 + }, + grid: { hoverable: true, + clickable: true, + tickColor: "#f9f9f9", + borderWidth: 0 + }, + colors: ["#3B5998"], + xaxis: {ticks:6, tickDecimals: 0}, + yaxis: {ticks:3, tickDecimals: 0}, + }); + + + + + + + }); diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 3d34101e..1fb6c1fd 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -255,7 +255,7 @@ arcFillEnd: 12, arcFillTotal: 20, - inc: 1.0, + inc: -0.7, incTot: 1.0, arcBgColorLight: 200, arcBgColorSat: 50, From 9c3b573af298daaa5c7e101fb4097d73f4c8e5df Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 12 Aug 2016 07:49:24 +0200 Subject: [PATCH 07/68] Splited js from html + Added some ID + small display modification. --- .../sentiment_analysis_trending.html | 129 +++--------------- 1 file changed, 19 insertions(+), 110 deletions(-) diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 1fb6c1fd..245af336 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -107,20 +107,20 @@
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • +
  • Graph 5
    • +
  • Graph 6
    • +
  • Graph 7
    • +
  • Graph 8
@@ -129,9 +129,11 @@
+
+
@@ -161,20 +163,20 @@
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • +
  • Graph 1
    • +
  • Graph 2
    • +
  • Graph 3
    • +
  • Graph 4
    -
  • Graph 1
    • -
  • Graph 2
    • -
  • Graph 3
    • -
  • Graph 4
    • +
  • Graph 5
    • +
  • Graph 6
    • +
  • Graph 7
    • +
  • Graph 8
@@ -238,106 +240,13 @@ + From 0431ecf434268061e1c59cb0c0658688f03c8d67 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 12 Aug 2016 10:22:24 +0200 Subject: [PATCH 08/68] Modified webpage display and added js. --- var/www/static/js/sentiment_trending.js | 140 ++++++++++++++++++ .../sentiment_analysis_trending.html | 56 ++++--- 2 files changed, 173 insertions(+), 23 deletions(-) create mode 100644 var/www/static/js/sentiment_trending.js diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js new file mode 100644 index 00000000..53f7da5a --- /dev/null +++ b/var/www/static/js/sentiment_trending.js @@ -0,0 +1,140 @@ + + +/* ---------------- Gauge ---------------- */ +var gaugeOptions = { + animateEasing: true, + + elementWidth: 200, + elementHeight: 125, + + arcFillStart: 10, + arcFillEnd: 12, + arcFillTotal: 20, + incTot: 1.0, + + arcBgColorLight: 200, + arcBgColorSat: 0, + arcStrokeFg: 20, + arcStrokeBg: 30, + + colorArcFg: '#FF3300', + animateSpeed: 1, + +}; +// Clone object +var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); +var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); + + + +gaugeOptions.appendTo = '#gauge_today_last_hour'; +gaugeOptions.dialLabel = 'Last hour'; +gaugeOptions.elementId = 'gauge1'; +gaugeOptions.inc = -0.7; +var gauge_today_last_hour = new FlexGauge(gaugeOptions); + +gaugeOptions2.appendTo = '#gauge_today_last_days'; +gaugeOptions2.dialLabel = 'Today'; +gaugeOptions2.elementId = 'gauge2'; +gaugeOptions2.inc = 0.4; +var gauge_today_last_days = new FlexGauge(gaugeOptions2); + +gaugeOptions3.appendTo = '#gauge_week'; +gaugeOptions3.dialLabel = 'Week'; +gaugeOptions3.elementId = 'gauge3'; +gaugeOptions3.inc = -0.3; +var gauge_today_last_days = new FlexGauge(gaugeOptions3); + + + + + + + + + +/* ---------- Sparkline Charts ---------- */ +//generate random number for charts +randNum = function(){ + var num = Math.random(); + if(num > 0.5) + num = -1+num; + console.log(Math.floor(num*101)); + return Math.floor(num*101); + //return (Math.floor( Math.random()* (1+40-20) ) ) + 20; +} + + + + var sparklineOptions = { + width: 250,//Width of the chart - Defaults to 'auto' - May be any valid css width - 1.5em, 20px, etc (using a number without a unit specifier won't do what you want) - This option does nothing for bar and tristate chars (see barWidth) + height: 80,//Height of the chart - Defaults to 'auto' (line height of the containing tag) + type: 'bar', + barSpacing: 0, + barWidth: 10, + barColor: '#00bf5f', + negBarColor: '#f22929', + zeroColor: '#ffff00' + }; + + +//sparklines (making loop with random data for all 10 sparkline) +i=1; +for (i=1; i<10; i++) { + var data = [3+randNum(), 5+randNum(), 8+randNum(), 11+randNum(),14+randNum(),17+randNum(),20+randNum(),15+randNum(),18+randNum(),22+randNum()]; + placeholder = '.sparkLineStatsToday' + i; + + $(placeholder).sparkline(data, sparklineOptions); + +} + +//sparklines (making loop with random data for all 10 sparkline) +i=1; +for (i=1; i<10; i++) { + var data = [3+randNum(), 5+randNum(), 8+randNum(), 11+randNum(),14+randNum(),17+randNum(),20+randNum(),15+randNum(),18+randNum(),22+randNum()]; + placeholder = '.sparkLineStatsWeek' + i; + + $(placeholder).sparkline(data, sparklineOptions); + +} + + +/* ----------- CanvasJS ------------ */ +var options_canvasJS = { + + animationEnabled: true, + axisY: { + tickThickness: 0, + lineThickness: 0, + valueFormatString: " ", + gridThickness: 0 + }, + axisX: { + tickThickness: 0, + lineThickness: 0, + labelFontSize: 0.1, + }, + data: [ + { + type: "bar", + color: "green", + dataPoints: [ + {y: 25} + ] + }, + { + type: "bar", + color: "red", + dataPoints: [ + {y: -13} + ] + } + ] +}; + +var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS); +var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS); + +chart_canvas1.render(); +chart_canvas2.render(); + diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 245af336..7cc730ef 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -13,8 +13,9 @@ - + + + + + +
+ +
+ +
+
+
+

Terms management interface

+
+ +
+ +
+ + +
+
+
+
+
+
+ Manage tracked terms +
+
+ +
+ + + +
+ + + + + + + + + + + + + + {% set i = 0 %} + {% for term in track_list %} + + + + + + + + + {% set i = i + 1 %} + {% endfor %} + +
Term nameDateDay occurenceWeek occurenceMonth occurenceAction
{{ term }}{{ i }}1555459

+ + +

+ +
+
+ +
+ +
+
+ + +
+
+
+
+
+ Manage blacklisted terms +
+
+ +
+ + + +
+ + + + + + + + + + + {% set i = 0 %} + {% for term in black_list %} + + + + + + {% set i = i + 1 %} + {% endfor %} + +
Term nameDateAction
{{ term }}{{ i }}

+ +

+ +
+
+ +
+ +
+
+ + +
+ +
+ + + + + + + + + diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html new file mode 100644 index 00000000..73121d57 --- /dev/null +++ b/var/www/templates/terms_plot_tool.html @@ -0,0 +1,150 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+

Terms plot tool

+
+ +
+ +
+ + +
+
+
+
+
+
+ Select options +
+
+
+ +
+ +
+
+
    +
+
+
+
+
+
    +
+
+
+
+ +
+
+ Date: +
+
+ +
+ + +
+
+ +
+
+ +
+ +
+
+ + +
+
+
+
+
+
+ Graph +
+
+
+
+ +
+
+ +
+ +
+
+ + + +
+ +
+ + + + + + + + From 8d41b109e7a7a54546eb88a04bc4c19354f665e4 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 19 Aug 2016 14:33:01 +0200 Subject: [PATCH 34/68] Added query of the data for the new tracked word --- var/www/Flask_server.py | 36 +++++++++++++++++++++++-- var/www/templates/terms_management.html | 14 +++++----- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 7ce34bdb..cb216447 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -218,6 +218,22 @@ def get_top_relevant_data(server, module_name): # member_set.insert(0, ("passed_days", days)) # return member_set + +def Term_getValueOverRange(word, startDate, num_day): + passed_days = 0 + oneDay = 60*60*24 + to_return = [] + curr_to_return = 0 + for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): + value = r_serv_term.hget(timestamp, word) + curr_to_return += int(value) if value is not None else 0 + for i in num_day: + if passed_days == i-1: + to_return.append(curr_to_return) + passed_days += 1 + return to_return + + # ========= CACHE CONTROL ======== @app.after_request def add_header(response): @@ -562,17 +578,33 @@ def terms_management(): TrackedTermsSet_Name = "TrackedSetTermSet" BlackListTermsSet_Name = "BlackListSetTermSet" + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + track_list = [] track_list_values = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): track_list.append(tracked_term) + track_list_values.append(Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])) + black_list = [] - black_list_values = [] for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): black_list.append(blacked_term) - return render_template("terms_management.html", black_list=black_list, track_list=track_list) + return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values) + + +@app.route("/terms_management_query/") +def terms_management_query(): + term = request.args.get('term') + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + print Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) + return jsonify(Term_getValueOverRange(term, today_timestamp, [1, 7, 31])) @app.route("/terms_management_action/", methods=['GET']) diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 7738f37d..629b8236 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -97,9 +97,9 @@ {{ term }} {{ i }} - 15 - 55 - 459 + {{ track_list_values[i][0] }} + {{ track_list_values[i][1] }} + {{ track_list_values[i][2] }}

@@ -211,9 +211,11 @@ function perform_operation(){ if(json.section == "followTerm") { if(json.action == "add") { // query data - var action_button = "" - table_track.row.add( [ json.term, 'now', 0, 0, 0, action_button ] ).draw( false ); - perform_binding(); + $.get("{{ url_for('terms_management_query') }}", { term: json.term }, function(data2, status){ + var action_button = "" + table_track.row.add( [ json.term, 'now', data2[0], data2[1], data2[2], action_button ] ).draw( false ); + perform_binding(); + }); } else if (json.action == "delete") { // Find indexes of row which have the term in the first column var index = table_track.row().eq( 0 ).filter( function (rowIdx) { From 0b6828fb16b5759de78f8a72c5cedbee7f30cb54 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 19 Aug 2016 16:53:46 +0200 Subject: [PATCH 35/68] Correctly handle and display add or remove a term in term-frequency + started sketch of terms-plot tool. --- bin/Curve.py | 8 ++++ var/www/Flask_server.py | 39 ++++++++++++++-- var/www/templates/terms_management.html | 41 +++++++++-------- var/www/templates/terms_plot_tool.html | 59 +++++++++++++------------ 4 files changed, 95 insertions(+), 52 deletions(-) diff --git a/bin/Curve.py b/bin/Curve.py index f804f2bd..e7435eb6 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -14,6 +14,14 @@ words and their occurency per day. ..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put the same Subscriber name in both of them. + + + +zrank for each day +week -> top zrank for each day + + + Requirements ------------ diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index cb216447..9c98642e 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -5,6 +5,7 @@ import redis import ConfigParser import json import datetime +import time import calendar from flask import Flask, render_template, jsonify, request import flask @@ -577,6 +578,8 @@ def sentiment_analysis_plot_tool_getdata(): def terms_management(): TrackedTermsSet_Name = "TrackedSetTermSet" BlackListTermsSet_Name = "BlackListSetTermSet" + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) @@ -586,32 +589,58 @@ def terms_management(): track_list_values = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): track_list.append(tracked_term) - track_list_values.append(Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])) + value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31]) + + term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + value_range.append(term_date) + track_list_values.append(value_range) black_list = [] for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): - black_list.append(blacked_term) + term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + print term_date + black_list.append([blacked_term, term_date]) return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values) @app.route("/terms_management_query/") def terms_management_query(): + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" term = request.args.get('term') + section = request.args.get('section') + today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) + value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) - print Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) - return jsonify(Term_getValueOverRange(term, today_timestamp, [1, 7, 31])) + if section == "followTerm": + term_date = r_serv_term.hget(TrackedTermsDate_Name, term) + elif section == "blacklistTerm": + term_date = r_serv_term.hget(BlackListTermsDate_Name, term) + + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + value_range.append(str(term_date)) + return jsonify(value_range) @app.route("/terms_management_action/", methods=['GET']) def terms_management_action(): TrackedTermsSet_Name = "TrackedSetTermSet" + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" BlackListTermsSet_Name = "BlackListSetTermSet" + today = datetime.datetime.now() + today = today.replace(microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + section = request.args.get('section') action = request.args.get('action') term = request.args.get('term') @@ -621,11 +650,13 @@ def terms_management_action(): if section == "followTerm": if action == "add": r_serv_term.sadd(TrackedTermsSet_Name, term) + r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp) else: r_serv_term.srem(TrackedTermsSet_Name, term) elif section == "blacklistTerm": if action == "add": r_serv_term.sadd(BlackListTermsSet_Name, term) + r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) else: r_serv_term.srem(BlackListTermsSet_Name, term) else: diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 629b8236..f4ebb0cb 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -83,8 +83,8 @@ - - + + @@ -96,7 +96,7 @@ {% for term in track_list %} - + @@ -137,8 +137,8 @@
Term nameDateTermAdded date Day occurence Week occurence Month occurence
{{ term }}{{ i }}{{ track_list_values[i][3] }} {{ track_list_values[i][0] }} {{ track_list_values[i][1] }} {{ track_list_values[i][2] }}
- - + + @@ -146,10 +146,10 @@ {% set i = 0 %} {% for term in black_list %} - - + + {% set i = i + 1 %} @@ -211,29 +211,32 @@ function perform_operation(){ if(json.section == "followTerm") { if(json.action == "add") { // query data - $.get("{{ url_for('terms_management_query') }}", { term: json.term }, function(data2, status){ + $.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ var action_button = "" - table_track.row.add( [ json.term, 'now', data2[0], data2[1], data2[2], action_button ] ).draw( false ); + table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], action_button ] ).draw( false ); perform_binding(); }); } else if (json.action == "delete") { // Find indexes of row which have the term in the first column - var index = table_track.row().eq( 0 ).filter( function (rowIdx) { - return table_track.cell( rowIdx, 0 ).data() == json.term ? true : false; + var index = table_track.rows().eq( 0 ).filter( function (rowIdx) { + return table_track.cell( rowIdx, 0 ).data() === json.term; } ); - table_track.row().remove(index).draw( false ); + table_track.rows(index).remove().draw( false ); } } else if(json.section == "blacklistTerm"){ if(json.action == "add") { - var action_button = "" - table_black.row.add( [ json.term, 'now', action_button ] ).draw( false ); - perform_binding(); + $.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ + console.log(data2); + var action_button = "" + table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false ); + perform_binding(); + }); } else if (json.action == "delete") { // Find indexes of row which have the term in the first column - var index = table_black.row().eq( 0 ).filter( function (rowIdx) { - return table_black.cell( rowIdx, 0 ).data() == json.term ? true : false; + var index = table_black.rows().eq( 0 ).filter( function (rowIdx) { + return table_black.cell( rowIdx, 0 ).data() === json.term; } ); - table_black.row().remove(index).draw( false ); + table_black.rows(index).remove().draw( false ); } } diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index 73121d57..c3373658 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -54,7 +54,7 @@
-

Terms plot tool

+

Terms plot tool

@@ -71,34 +71,15 @@ Select options
-
- -
- -
-
-
    -
-
-
-
-
-
    -
-
-
-
- -
-
- Date: -
-
- -
- - +
+ Date: + +
+ + +
+
@@ -142,9 +123,29 @@ $(document).ready(function(){ activePage = $('h1.page-header').attr('data-page'); $("#"+activePage).addClass("active"); + + + /* Create the slider and button*/ + var today = Date.now(); + var old_day = today - (12*31*24*60*60)*1000; + $( ".sliderRange" ).slider({ + range: true, + min: old_day, + max: today, + values: [ today - (7*24*60*60)*1000, today ], + step: 24*60*60*1000, + slide: function( event, ui ) { + $( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() ); + } + }); + +$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() + + " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() ); + +$('#plot_btn').click(plotData); + }); - From a5117d7cad1f0ee8a0a3093e9042447ea2216302 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 11:46:22 +0200 Subject: [PATCH 36/68] Added top set webpage --- var/www/Flask_server.py | 34 ++ var/www/templates/header.html | 3 +- var/www/templates/terms_management.html | 2 +- var/www/templates/terms_plot_top.html | 424 ++++++++++++++++++++++++ 4 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 var/www/templates/terms_plot_top.html diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 9c98642e..be634e77 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -227,6 +227,8 @@ def Term_getValueOverRange(word, startDate, num_day): curr_to_return = 0 for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): value = r_serv_term.hget(timestamp, word) + print timestamp, word + print value curr_to_return += int(value) if value is not None else 0 for i in num_day: if passed_days == i-1: @@ -675,6 +677,37 @@ def terms_plot_tool(): return render_template("terms_plot_tool.html") +@app.route("/terms_plot_top/") +def terms_plot_top(): + return render_template("terms_plot_top.html") + + +@app.route("/terms_plot_top_data/") +def terms_plot_top_data(): + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + the_set = request.args.get('set') + num_day = int(request.args.get('num_day')) + if the_set is None: + return "None" + else: + oneDay = 60*60*24 + to_return = [] + for term in r_serv_term.smembers(the_set): + value_range = [] + tot_sum = 0 + for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): + value = r_serv_term.hget(timestamp, term) + curr_value_range = int(value) if value is not None else 0 + tot_sum += curr_value_range + value_range.append([timestamp, curr_value_range]) + + to_return.append([term, value_range, tot_sum]) + + return jsonify(to_return) + @app.route("/test/") #completely shows the paste in a new tab def test(): @@ -700,6 +733,7 @@ def test(): # for e in array2: # stri += "

"+ e[0] + "\t" + str(e[1]) +"

" + print stri return stri diff --git a/var/www/templates/header.html b/var/www/templates/header.html index 07e1bdb7..4deb4740 100644 --- a/var/www/templates/header.html +++ b/var/www/templates/header.html @@ -15,7 +15,8 @@ diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index f4ebb0cb..02345fca 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -56,7 +56,7 @@
-

Terms management interface

+

Terms frequency: Management interface

diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html new file mode 100644 index 00000000..f17e4f37 --- /dev/null +++ b/var/www/templates/terms_plot_top.html @@ -0,0 +1,424 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+

Terms frequency: Top set information

+
+ +
+ +
+ + +
+
+
+
+
+
+ Today +
+
+
+
+ Today top word +
+
+
+
Term nameDateTermxAdded date Action
{{ term }}{{ i }}{{ black_list[i][0] }}{{ black_list[i][1] }}

- +

+ + + + + + + + + + +
TermValueActionDisplayed
+

+
+ + + + + + + + + + + +
TermValueActionDisplayed
+
+ + + +
+
+ Graph +
+
+
+
+ +
+ + + + + + + + + + + + +
+
+
+
+
+
+ Week +
+
+
+
+ Week top word +
+
+
+ + + + + + + + + + + +
TermValueActionDisplayed
+
+
+ + + + + + + + + + + +
TermValueActionDisplayed
+
+
+ +
+
+
+ Graph +
+
+
+
+ +
+ +
+ +
+
+ +
+ +
+
+ + +
+
+
+
+
+
+ Month +
+
+
+
+ Month top word +
+
+
+ + + + + + + + + + + +
TermValueActionDisplayed
+
+
+ + + + + + + + + + + +
TermValueActionDisplayed
+
+ +
+ +
+
+
+ Graph +
+
+
+
+ +
+ +
+ +
+
+ +
+ +
+
+ + + + + + + + + + + + + + + + + From fb3dea6bd2195ea135f1eac4978936f6d6985675 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 15:59:22 +0200 Subject: [PATCH 37/68] Finnished terms-plot-top-set. Also added possibility to add terms to track or blacklist. --- var/www/Flask_server.py | 12 +++++ var/www/templates/terms_plot_tool.html | 61 ++++++++++++++++++++--- var/www/templates/terms_plot_top.html | 67 ++++++++++++++++++++------ 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index be634e77..8a2d4d37 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -677,6 +677,18 @@ def terms_plot_tool(): return render_template("terms_plot_tool.html") +@app.route("/terms_plot_tool_data/") +def terms_plot_tool_data(): + num_day = int(request.args.get('num_day')) + term = request.args.get('term') + + if term is None: + print 'cc' + + + return jsonify() + + @app.route("/terms_plot_top/") def terms_plot_top(): return render_template("terms_plot_top.html") diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index c3373658..d663eb60 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -76,8 +76,8 @@
- - + +
@@ -139,13 +139,62 @@ } }); -$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() + - " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() ); - -$('#plot_btn').click(plotData); + $( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() + + " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() ); + + $('#plot-btn').click(plotData); }); + + + diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index f17e4f37..d7c4c8c4 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -307,6 +307,10 @@ set_today = "TopTermFreq_set_day"; set_week = "TopTermFreq_set_week"; set_month = "TopTermFreq_set_month"; +var plot_today; +var plot_week; +var plot_month; + var promises = []; // Used to know when everything has been received promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today, num_day: 1 }, function(data, status){ @@ -323,15 +327,14 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table_today.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-today", data[i][0])+"") + table_today.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0])+"") else - table_today2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-today", data[i][0])+"") + table_today2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0])+"") } - var plot_today = $.plot($("#graph-today"), to_plot, graph_options); + plot_today = $.plot($("#graph-today"), to_plot, graph_options); })); promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, num_day: 7 }, function(data, status){ - console.log(data); data.sort(function(a, b){return b[2]-a[2];}); // Sort data var table = $("#table-week") @@ -344,15 +347,14 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-week", data[i][0])+"") + table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0])+"") else - table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-week", data[i][0])+"") + table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0])+"") } - var plot_week = $.plot($("#graph-week"), to_plot, graph_options); + plot_week = $.plot($("#graph-week"), to_plot, graph_options); })); promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month, num_day: 31 }, function(data, status){ - console.log(data); data.sort(function(a, b){return b[2]-a[2];}); // Sort data var table = $("#table-month") @@ -365,11 +367,11 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-month", data[i][0])+"") + table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0])+"") else - table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("graph-month", data[i][0])+"") + table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0])+"") } - var plot_month = $.plot($("#graph-month"), to_plot, graph_options); + plot_month = $.plot($("#graph-month"), to_plot, graph_options); })); /* When everything has been received, start adding tooltip */ @@ -384,9 +386,9 @@ $.when.apply($, promises).done( function (arg) { function addbuttons(term) { - return ""+ - ""; } @@ -401,7 +403,6 @@ function perform_operation(){ var data_to_send = { section: curr_section, action:"add", term: curr_term}; - console.log(data_to_send); $.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){ if(status == "success") { location.reload(); @@ -414,6 +415,44 @@ function hide_or_show() { var graph = $(this).attr('data-graph'); var checked = $(this).prop('checked') console.log(curr_term + ' ' + graph + ' ' + checked); + + if(graph == "today") { + var graphData = plot_today.getData(); + var index; + for(i=0; i Date: Sat, 20 Aug 2016 16:23:06 +0200 Subject: [PATCH 38/68] Added terms-plot tool --- var/www/Flask_server.py | 20 +++++++++++++++----- var/www/templates/terms_plot_tool.html | 14 +++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 8a2d4d37..a3c18ebc 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -679,14 +679,24 @@ def terms_plot_tool(): @app.route("/terms_plot_tool_data/") def terms_plot_tool_data(): - num_day = int(request.args.get('num_day')) + oneDay = 60*60*24 + range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; + range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) + range_start = calendar.timegm(range_start.timetuple()) + range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; + range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) + range_end = calendar.timegm(range_end.timetuple()) term = request.args.get('term') if term is None: - print 'cc' - - - return jsonify() + return "None" + else: + value_range = [] + for timestamp in range(range_start, range_end+oneDay, oneDay): + value = r_serv_term.hget(timestamp, term) + curr_value_range = int(value) if value is not None else 0 + value_range.append([timestamp, curr_value_range]) + return jsonify(value_range) @app.route("/terms_plot_top/") diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index d663eb60..4ed03a74 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -177,17 +177,17 @@ var graph_options = { function plotData(){ var curthis = $(this); var term = $('#TermInput').val(); - console.log(term); + var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000; + var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000; - $.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month, num_day: 31 }, function(data, status){ + $.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){ var to_plot = []; + console.log(data); + var curr_data = []; for(i=0; i Date: Sat, 20 Aug 2016 16:38:51 +0200 Subject: [PATCH 39/68] Enter simulate button click on input field for terms management and plot tool --- var/www/templates/terms_management.html | 23 ++++++++++++++++++++--- var/www/templates/terms_plot_tool.html | 7 +++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 02345fca..61561fc7 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -76,8 +76,8 @@
- - + +
@@ -131,7 +131,7 @@
- +
@@ -182,11 +182,28 @@ table_track = $('#myTable').DataTable(); table_black = $('#myTable2').DataTable(); + $("#followTermInput").keyup(function(event){ + if(event.keyCode == 13){ + $("#followTermBtn").click(); + $("#followTermInput").val(""); + } + }); + + $("#blacklistTermInput").keyup(function(event){ + if(event.keyCode == 13){ + $("#blacklistTermBtn").click(); + $("#blacklistTermInput").val(""); + } + }); + perform_binding(); }); From 3f1cf705fcc00798abab838863ff9494e406d248 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 16:41:49 +0200 Subject: [PATCH 40/68] Deleted alert() --- var/www/templates/terms_plot_top.html | 1 - 1 file changed, 1 deletion(-) diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index d7c4c8c4..366422ef 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -397,7 +397,6 @@ function addcheckbox(graph, term) { } function perform_operation(){ - alert("are u sure?"); var curr_section = $(this).attr('data-section'); var curr_term = $(this).attr('data-term'); From 2ed60cb9bf87e85c4af5147f1aa06943a45c7080 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 16:44:36 +0200 Subject: [PATCH 41/68] Fixed typo --- var/www/templates/terms_plot_tool.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index 1be1fd1a..79ddb32a 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -54,7 +54,7 @@
-

Terms plot tool

+

Terms plot tool

From ee8f26c7c4f357b39d38708657a90239ab38144b Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 18:14:19 +0200 Subject: [PATCH 42/68] In sentiment trending, if there are no providers in the today\'s set, get data from random providers --- var/www/Flask_server.py | 8 +++++++- var/www/static/js/indexjavascript.js | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index a3c18ebc..4a8cbb1a 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -509,7 +509,13 @@ def sentiment_analysis_getplotdata(): dateStart_timestamp = calendar.timegm(dateStart.timetuple()) to_return = {} - for cur_provider in r_serv_charts.zrangebyscore('providers_set_'+ get_date_range(0)[0], '-inf', '+inf', start=0, num=8): + range_providers = r_serv_charts.zrangebyscore('providers_set_'+ get_date_range(1)[1], '-inf', '+inf', start=0, num=8) + if range_providers == []: + print 'today provider empty' + range_providers = r_serv_charts.smembers('all_provider_set') + + for cur_provider in range_providers: + print cur_provider cur_provider_name = cur_provider + '_' list_date = {} for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index c936e9c7..2eb7296e 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -309,10 +309,12 @@ $(document).ready(function () { refresh(); update_values(); - if($('#button-toggle-queues').prop('checked')) + if($('#button-toggle-queues').prop('checked')){ create_queue_table(); - else + } + else{ $("#queueing").html(''); + } for (i = 0; i < (glob_tabvar.row1).length; i++){ From 841e87eb16a7f5d5b25ef71e0e38bfadea96d0d8 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Sat, 20 Aug 2016 18:48:24 +0200 Subject: [PATCH 43/68] Fixed bug in sentiment trending, if there was no data for the avg -> there was one NaN leading to no display in the gauge. --- var/www/Flask_server.py | 2 +- var/www/static/js/sentiment_trending.js | 6 +++--- var/www/templates/sentiment_analysis_trending.html | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 4a8cbb1a..3f434067 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -509,7 +509,7 @@ def sentiment_analysis_getplotdata(): dateStart_timestamp = calendar.timegm(dateStart.timetuple()) to_return = {} - range_providers = r_serv_charts.zrangebyscore('providers_set_'+ get_date_range(1)[1], '-inf', '+inf', start=0, num=8) + range_providers = r_serv_charts.zrangebyscore('providers_set_'+ get_date_range(0)[0], '-inf', '+inf', start=0, num=8) if range_providers == []: print 'today provider empty' range_providers = r_serv_charts.smembers('all_provider_set') diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index 1d69f6ed..2caf91d4 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -42,7 +42,6 @@ $.getJSON("/sentiment_analysis_getplotdata/", function(data) { - console.log(data); var all_data = []; var plot_data = []; var graph_avg = []; @@ -120,6 +119,8 @@ $.getJSON("/sentiment_analysis_getplotdata/", all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal; var curr_avg = curr_sum / (curr_sum_elem); + if(isNaN(curr_avg)) + curr_avg = 0.0 //var curr_avg = curr_sum / (oneWeek/oneHour); //var curr_avg = curr_sum / (spark_data.length); graph_avg.push([curr_provider, curr_avg]); @@ -134,6 +135,7 @@ $.getJSON("/sentiment_analysis_getplotdata/", // print week var num = graphNum + 1; var placeholder = '.sparkLineStatsWeek' + num; + sparklineOptions.barWidth = 2; $(placeholder).sparkline(plot_data[graphNum], sparklineOptions); $(placeholder+'t').text(curr_provider); var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); @@ -143,7 +145,6 @@ $.getJSON("/sentiment_analysis_getplotdata/", sparklineOptions.tooltipFormat = '● Avg: {{value}} ' $(placeholder+'b').sparkline([curr_avg], sparklineOptions); sparklineOptions.tooltipFormat = '● {{offset:names}}, {{value}} ' - sparklineOptions.barWidth = 2; sparklineOptions.tooltipValueLookups = { names: offset_to_time}; sparklineOptions.chartRangeMax = max_value_day; @@ -180,7 +181,6 @@ $.getJSON("/sentiment_analysis_getplotdata/", all_day_avg_maxVal = Math.abs(day_avg) > all_day_avg_maxVal ? Math.abs(day_avg) : all_day_avg_maxVal; $(placeholder+'b').sparkline([day_avg], sparklineOptions); sparklineOptions.tooltipFormat = '● {{offset:names}}, {{value}} ' - sparklineOptions.barWidth = 2; $(placeholder+'s').text(day_avg_text); avgName = ".pannelToday" + num; diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 89b4f675..3a927251 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -21,12 +21,12 @@ .moodtable_worst { background: rgba(255, 0, 0, 0.47); - font-size: unset; + font-size: small; } .moodtable_best { background: rgba(132, 255, 0, 0.5); - font-size: unset; + font-size: small; } .jqstooltip{ From d3c0fc5f94992a5a75cd0bb83d843b5b277ce44c Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 13:35:49 +0200 Subject: [PATCH 44/68] Added terms trending module and web interface: - Curve module has been rewritten - Added new module curve_manage_top_set - Added all web-pages --- bin/Curve.py | 96 +++++++++--------- bin/Curve_manage_top_sets.py | 128 ++++++++++++++++++++++++ bin/LAUNCH.sh | 10 +- bin/packages/modules.cfg | 4 + var/www/Flask_server.py | 66 ++++++++++-- var/www/templates/terms_management.html | 90 ++++++++++++++++- var/www/templates/terms_plot_tool.html | 5 +- var/www/templates/terms_plot_top.html | 55 +++++++++- 8 files changed, 387 insertions(+), 67 deletions(-) create mode 100755 bin/Curve_manage_top_sets.py diff --git a/bin/Curve.py b/bin/Curve.py index e7435eb6..b43fcb1d 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -41,11 +41,25 @@ import calendar from Helper import Process # Config Variables +BlackListTermsSet_Name = "BlackListSetTermSet" +TrackedTermsSet_Name = "TrackedSetTermSet" top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set +oneDay = 60*60*24 +top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] +top_termFreq_setName_week = ["TopTermFreq_set_week", 7] +top_termFreq_setName_month = ["TopTermFreq_set_month", 31] +top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] + + +def check_if_tracked_term(term, path): + if term in TrackedTermsSet_Name: + #add_paste to tracked_word_set + set_name = "tracked_" + term + server.sadd(set_name, path) + p.populate_set_out("New Term added", 'CurveManageTopSets') def getValueOverRange(word, startDate, num_day): - oneDay = 60*60*24 to_return = 0 for timestamp in range(startDate, startDate - num_day*oneDay, -oneDay): value = server_term.hget(timestamp, word) @@ -86,10 +100,9 @@ if __name__ == "__main__": generate_new_graph = False # Term Frequency - top_termFreq_setName_day = ["TopTermFreq_set_day", 1] + top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] - top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] while True: @@ -100,60 +113,51 @@ if __name__ == "__main__": temp = filename.split('/') date = temp[-4] + temp[-3] + temp[-2] timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) - - # If set size is greater then the one authorized - # suppress smaller elements - for curr_set, curr_num_day in top_termFreq_set_array: - diffCard = server_term.scard(curr_set) - top_term_freq_max_set_cardinality - if diffCard > 0: - top_termFreq = server_term.smembers(curr_set) - sorted_top_termFreq_set = [] - for word in top_termFreq: - word_value = getValueOverRange(word, timestamp, curr_num_day) - sorted_top_termFreq_set.append((word, word_value)) - - sorted_top_termFreq_set.sort(key=lambda tup: tup[1]) - for i in range(0, diffCard): - print 'set oversized, dropping', sorted_top_termFreq_set[i][0] - server_term.srem(curr_set, sorted_top_termFreq_set[i][0]) + top_termFreq_setName_day[0] += timestamp - #timer = time.clock() low_word = word.lower() - #print 'wordlower', time.clock() - timer + #Old curve r_serv1.hincrby(low_word, date, int(score)) # Update redis curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score))) + if low_word not in server.smembers(BlackListTermsSet_Name): + server.zincrby(top_termFreq_setName_day[0], int(score), low_word) - # Manage Top set - for curr_set, curr_num_day in top_termFreq_set_array: + #Add more info for tracked terms + check_if_tracked_term(low_word, filename) - if server_term.scard(curr_set) < top_term_freq_max_set_cardinality: + # Manage Top set is done in module Curve_manage_top_sets + + ''' + if server_term.scard(curr_set) < top_term_freq_max_set_cardinality: + server_term.sadd(curr_set, low_word) + elif server_term.sismember(curr_set, low_word): + continue + + else: + + + #timer = time.clock() + curr_word_value = getValueOverRange(low_word, timestamp, curr_num_day) + #print 'curr_range', time.clock() - timer + top_termFreq = server_term.smembers(curr_set) + sorted_top_termFreq_set = [] + #timer = time.clock() + for word in top_termFreq: + word_value = getValueOverRange(word, timestamp, curr_num_day) + sorted_top_termFreq_set.append((word, word_value)) + + sorted_top_termFreq_set.sort(key=lambda tup: tup[1]) + #print 'whole_range', time.clock() - timer + + if curr_word_value > int(sorted_top_termFreq_set[0][1]): + print str(curr_num_day)+':', low_word, curr_word_value, '\t', sorted_top_termFreq_set[0][0], sorted_top_termFreq_set[0][1], '\t', curr_word_value > sorted_top_termFreq_set[0][1] + server_term.srem(curr_set, sorted_top_termFreq_set[0][0]) server_term.sadd(curr_set, low_word) - elif server_term.sismember(curr_set, low_word): - continue - - else: - #timer = time.clock() - curr_word_value = getValueOverRange(low_word, timestamp, curr_num_day) - #print 'curr_range', time.clock() - timer - top_termFreq = server_term.smembers(curr_set) - sorted_top_termFreq_set = [] - #timer = time.clock() - for word in top_termFreq: - word_value = getValueOverRange(word, timestamp, curr_num_day) - sorted_top_termFreq_set.append((word, word_value)) - - sorted_top_termFreq_set.sort(key=lambda tup: tup[1]) - #print 'whole_range', time.clock() - timer - - if curr_word_value > int(sorted_top_termFreq_set[0][1]): - print str(curr_num_day)+':', low_word, curr_word_value, '\t', sorted_top_termFreq_set[0][0], sorted_top_termFreq_set[0][1], '\t', curr_word_value > sorted_top_termFreq_set[0][1] - server_term.srem(curr_set, sorted_top_termFreq_set[0][0]) - server_term.sadd(curr_set, low_word) - + ''' else: if generate_new_graph: generate_new_graph = False diff --git a/bin/Curve_manage_top_sets.py b/bin/Curve_manage_top_sets.py new file mode 100755 index 00000000..c3494c16 --- /dev/null +++ b/bin/Curve_manage_top_sets.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + + + +zrank for each day +week -> top zrank for each day + + +Requirements +------------ + +*Need running Redis instances. (Redis) +*Categories files of words in /files/ need to be created +*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly. + +""" + +import redis +import time +import copy +from pubsublogger import publisher +from packages import lib_words +import os +import datetime +import calendar + +from Helper import Process + +# Config Variables +BlackListTermsSet_Name = "BlackListSetTermSet" +TrackedTermsSet_Name = "TrackedSetTermSet" +top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set +oneDay = 60*60*24 +num_day_month = 31 +num_day_week = 7 + +top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] +top_termFreq_setName_week = ["TopTermFreq_set_week", 7] +top_termFreq_setName_month = ["TopTermFreq_set_month", 31] +top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] + + +def manage_top_set(): + startDate = datetime.datetime.now() + startDate = startDate.replace(hour=0, minute=0, second=0, microsecond=0) + startDate = calendar.timegm(startDate.timetuple()) + + dico = {} + + # Retreive top data (2*max_card) from days sets + for timestamp in range(startDate, startDate - num_day_month*oneDay, -oneDay): + top_termFreq_setName_day[0] += str(timestamp) + array_top_day = server_term.zrangebyscore(top_termFreq_setName_day[0], '-inf', '+inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2) + + for word, value in array_top_day: + if word in dico.keys(): + dico[word] += value + else: + dico[word] = value + + if timestamp == startDate - num_day_week*oneDay: + dico_week = copy.deepcopy(dico) + + # convert dico into sorted array + array_month = [] + for w, v in dico.iteritems(): + array_month.append((w, v)) + array_month.sort(key=lambda tup: -tup[1]) + + array_week = [] + for w, v in dico_week.iteritems(): + array_week.append((w, v)) + array_week.sort(key=lambda tup: -tup[1]) + + + # suppress every terms in top sets + for curr_set, curr_num_day in top_termFreq_set_array[1:3]: + for w in server_term.zrange(curr_set, 0, -1): + server_term.zrem(curr_set, w) + + # Add top term from sorted array in their respective sorted sets + for elem in array_week: + server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0]) + + for elem in array_month: + server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0]) + + + + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + config_section = 'CurveManageTopSets' + p = Process(config_section) + + server_term = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_TermFreq", "host"), + port=p.config.get("Redis_Level_DB_TermFreq", "port"), + db=p.config.get("Redis_Level_DB_TermFreq", "db")) + + # FUNCTIONS # + publisher.info("Script Curve_manage_top_set started") + + # Sent to the logging a description of the module + publisher.info("Manage the top sets with the data created by the module curve.") + + manage_top_set() + + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + print 'sleeping' + time.sleep(60) # sleep a long time then manage the set + manage_top_set() + continue + + # Do something with the message from the queue + #manage_top_set() + diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 62700d1f..fef9c3ce 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -136,7 +136,9 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' sleep 0.1 - #screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' + screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' sleep 0.1 @@ -158,12 +160,6 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' } #If no params, display the help diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 0f3ceb38..7434def8 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -24,6 +24,10 @@ publish = Redis_Words [Curve] subscribe = Redis_Words +publish = Redis_CurveManageTopSets + +[CurveManageTopSets] +subscribe = Redis_CurveManageTopSets [Categ] subscribe = Redis_Global diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 3f434067..56f43907 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -227,8 +227,8 @@ def Term_getValueOverRange(word, startDate, num_day): curr_to_return = 0 for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): value = r_serv_term.hget(timestamp, word) - print timestamp, word - print value + #print timestamp, word + #print value curr_to_return += int(value) if value is not None else 0 for i in num_day: if passed_days == i-1: @@ -595,11 +595,15 @@ def terms_management(): track_list = [] track_list_values = [] + track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): track_list.append(tracked_term) value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31]) term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) + + set_paste_name = "tracked_" + tracked_term + track_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" value_range.append(term_date) track_list_values.append(value_range) @@ -609,10 +613,38 @@ def terms_management(): for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - print term_date black_list.append([blacked_term, term_date]) - return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values) + return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste) + + +@app.route("/terms_management_query_paste/") +def terms_management_query_paste(): + term = request.args.get('term') + TrackedTermsSet_Name = "TrackedSetTermSet" + paste_info = [] + + set_paste_name = "tracked_" + term + track_list_path = r_serv_term.smembers(set_paste_name) + print set_paste_name + print track_list_path + + for path in track_list_path: + paste = Paste.Paste(path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_language = paste._get_p_language() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content().decode('utf-8', 'ignore') + if p_content != 0: + p_content = p_content[0:400] + paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "language": p_language, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) + + return jsonify(paste_info) @app.route("/terms_management_query/") @@ -680,7 +712,11 @@ def terms_management_action(): @app.route("/terms_plot_tool/") def terms_plot_tool(): - return render_template("terms_plot_tool.html") + term = request.args.get('term') + if term is not None: + return render_template("terms_plot_tool.html", term=term) + else: + return render_template("terms_plot_tool.html", term="") @app.route("/terms_plot_tool_data/") @@ -699,6 +735,7 @@ def terms_plot_tool_data(): else: value_range = [] for timestamp in range(range_start, range_end+oneDay, oneDay): + print timestamp, term value = r_serv_term.hget(timestamp, term) curr_value_range = int(value) if value is not None else 0 value_range.append([timestamp, curr_value_range]) @@ -712,6 +749,7 @@ def terms_plot_top(): @app.route("/terms_plot_top_data/") def terms_plot_top_data(): + oneDay = 60*60*24 today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) @@ -721,7 +759,22 @@ def terms_plot_top_data(): if the_set is None: return "None" else: - oneDay = 60*60*24 + to_return = [] + if the_set == "TopTermFreq_set_day": + the_set += "_" + str(today_timestamp) + + for term, tot_value in r_serv_term.zrangebyscore(the_set, '-inf', '+inf', withscores=True, start=0, num=20): + value_range = [] + for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): + value = r_serv_term.hget(timestamp, term) + curr_value_range = int(value) if value is not None else 0 + value_range.append([timestamp, curr_value_range]) + + to_return.append([term, value_range, tot_value]) + + return jsonify(to_return) + + ''' to_return = [] for term in r_serv_term.smembers(the_set): value_range = [] @@ -735,6 +788,7 @@ def terms_plot_top_data(): to_return.append([term, value_range, tot_sum]) return jsonify(to_return) + ''' @app.route("/test/") #completely shows the paste in a new tab diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 61561fc7..4dc5a05e 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -33,6 +33,24 @@ + + +
+ @@ -100,9 +119,10 @@ + {% set i = i + 1 %} @@ -197,6 +217,66 @@ }); perform_binding(); + + // On click, get html content from url and update the corresponding modal + $("[data-toggle='modal']").on("click.openmodal", function (event) { + event.preventDefault(); + var the_modal=$(this); + var url = "{{ url_for('terms_management_query_paste') }}?term=" + $(this).attr('data-term'); + $.getJSON(url, function (data) { + //console.log(data); + if (data.length != 0) { + for (i=0; i"; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + html_to_add += ""; + + html_to_add += ""; + html_to_add += ""; + html_to_add += "
Day occurence Week occurence Month occurence# Concerned pastes Action
{{ track_list_values[i][0] }} {{ track_list_values[i][1] }} {{ track_list_values[i][2] }}{{ track_list_num_of_paste[i] }}

- - + +

SourceDateEncodingLanguageSize (Kb)MimeNumber of linesMax line lengthPreview
"+curr_data.source+""+curr_data.date+""+curr_data.encoding+""+curr_data.language+""+curr_data.size+""+curr_data.mime+""+curr_data.lineinfo[0]+""+curr_data.lineinfo[1]+"
"; + $("#mymodalbody").html(html_to_add); + $("[data-toggle=popover]").popover(); + $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); + } + } else { + $("#mymodalbody").html("No paste containing this term has been received yet."); + $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); + } + }); + + }); + + $("#mymodal").on('hidden.bs.modal', function () { + $("#mymodalbody").html("

Loading paste information...

"); + var loading_gif = ""; + $("#mymodalbody").append(loading_gif); // Show the loading GIF + }); + + }); @@ -229,8 +309,8 @@ function perform_operation(){ if(json.action == "add") { // query data $.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ - var action_button = "" - table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], action_button ] ).draw( false ); + var action_button = "" + table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false ); perform_binding(); }); } else if (json.action == "delete") { diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index 79ddb32a..7e20fe7a 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -76,7 +76,7 @@
- +
@@ -144,6 +144,9 @@ $('#plot-btn').click(plotData); + $("#TermInput").val($("#TermInput").attr("data-init-plot")); + $("#plot-btn").click(); + $("#TermInput").keyup(function(event){ if(event.keyCode == 13){ $("#plot-btn").click(); diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index 366422ef..da5b9b2f 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -266,6 +266,7 @@ + diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index da5b9b2f..da3611bf 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -463,11 +463,9 @@ function hide_or_show() { var curr_term = $(this).attr('data-term'); var graph = $(this).attr('data-graph'); var checked = $(this).prop('checked') - console.log(curr_term + ' ' + graph + ' ' + checked); if(graph == "today") { var graphData = plot_today.getData(); - console.log(graphData); var index; for(i=0; i Date: Mon, 22 Aug 2016 16:01:42 +0200 Subject: [PATCH 47/68] Fixed bug with redis where the inserted key was too long, causing a crash + fixed bug taking min instead of max in terms-top-set --- bin/Curve.py | 8 ++++---- bin/Curve_manage_top_sets.py | 16 ++++++---------- var/www/Flask_server.py | 2 +- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/bin/Curve.py b/bin/Curve.py index 641480eb..0a533a35 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -55,7 +55,7 @@ def check_if_tracked_term(term, path): if term in TrackedTermsSet_Name: #add_paste to tracked_word_set set_name = "tracked_" + term - server.sadd(set_name, path) + server_term.sadd(set_name, path) p.populate_set_out("New Term added", 'CurveManageTopSets') @@ -113,7 +113,7 @@ if __name__ == "__main__": temp = filename.split('/') date = temp[-4] + temp[-3] + temp[-2] timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) - top_termFreq_setName_day[0] += str(timestamp) + curr_set = top_termFreq_setName_day[0] + str(timestamp) low_word = word.lower() @@ -123,8 +123,8 @@ if __name__ == "__main__": # Update redis curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score))) - if low_word not in server.smembers(BlackListTermsSet_Name): - server.zincrby(top_termFreq_setName_day[0], int(score), low_word) + if low_word not in server_term.smembers(BlackListTermsSet_Name): + server_term.zincrby(curr_set, low_word, float(score)) #Add more info for tracked terms check_if_tracked_term(low_word, filename) diff --git a/bin/Curve_manage_top_sets.py b/bin/Curve_manage_top_sets.py index 167aa261..92d63b33 100755 --- a/bin/Curve_manage_top_sets.py +++ b/bin/Curve_manage_top_sets.py @@ -52,15 +52,14 @@ def manage_top_set(): # Retreive top data (2*max_card) from days sets for timestamp in range(startDate, startDate - top_termFreq_setName_month[1]*oneDay, -oneDay): curr_set = top_termFreq_setName_day[0] + str(timestamp) - print top_termFreq_setName_day[0] - array_top_day = server_term.zrangebyscore(curr_set, '-inf', '+inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2) + array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2) - print array_top_day for word, value in array_top_day: - if word in dico.keys(): - dico[word] += value - else: - dico[word] = value + if word not in server_term.smembers(BlackListTermsSet_Name): + if word in dico.keys(): + dico[word] += value + else: + dico[word] = value if timestamp == startDate - num_day_week*oneDay: dico_week = copy.deepcopy(dico) @@ -78,9 +77,6 @@ def manage_top_set(): array_week.sort(key=lambda tup: -tup[1]) array_week = array_week[0:20] - print array_month - print array_week - # suppress every terms in top sets for curr_set, curr_num_day in top_termFreq_set_array[1:3]: for w in server_term.zrange(curr_set, 0, -1): diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index d605c118..07001f9b 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -760,7 +760,7 @@ def terms_plot_top_data(): if the_set == "TopTermFreq_set_day": the_set += "_" + str(today_timestamp) - for term, tot_value in r_serv_term.zrangebyscore(the_set, '-inf', '+inf', withscores=True, start=0, num=20): + for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): value_range = [] for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): value = r_serv_term.hget(timestamp, term) From a424069c6422ad07304106297d81d8a704d4789d Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 16:12:20 +0200 Subject: [PATCH 48/68] Fixed bug in flask with zrange (was taking the bottom of the set instead of the top leading to get only the min instead of max) --- var/www/Flask_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 07001f9b..83e2e033 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -350,7 +350,7 @@ def providersChart(): redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_' redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0] - member_set = r_serv_charts.zrangebyscore(redis_provider_name_set, '-inf', '+inf', withscores=True, start=0, num=8) + member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8) # Member set is a list of (value, score) pairs if len(member_set) == 0: member_set.append(("No relevant data", float(100))) @@ -509,7 +509,7 @@ def sentiment_analysis_getplotdata(): dateStart_timestamp = calendar.timegm(dateStart.timetuple()) to_return = {} - range_providers = r_serv_charts.zrangebyscore('providers_set_'+ get_date_range(0)[0], '-inf', '+inf', start=0, num=8) + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) if range_providers == []: print 'today provider empty' range_providers = r_serv_charts.smembers('all_provider_set') From 27ae577e00f3ff6a207d7b9cfa20092487386a1a Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 16:25:52 +0200 Subject: [PATCH 49/68] Fixed stupid line remove in index.html... --- var/www/static/js/indexjavascript.js | 1 + 1 file changed, 1 insertion(+) diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 2eb7296e..f0cd101f 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -90,6 +90,7 @@ var source = new EventSource('/_logs'); source.onmessage = function(event) { var feed = jQuery.parseJSON( event.data ); + create_log_table(feed); }; function pad_2(number) From 2fd2c8af39bcb871fcd9817c89724893427bd664 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 20:59:56 +0200 Subject: [PATCH 50/68] Removed commented code --- bin/Curve.py | 44 ++-------- bin/Curve_manage_top_sets.py | 17 ++-- bin/ModuleStats.py | 85 +----------------- var/www/Flask_server.py | 87 +------------------ var/www/static/js/FlexGauge.js | 2 + .../templates/important_paste_by_module.html | 1 - 6 files changed, 16 insertions(+), 220 deletions(-) diff --git a/bin/Curve.py b/bin/Curve.py index 0a533a35..9337bc5b 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -1,9 +1,6 @@ #!/usr/bin/env python2 # -*-coding:UTF-8 -* """ -The ZMQ_Sub_Curve Module -============================ - This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module. This modules update a .csv file used to draw curves representing selected @@ -15,11 +12,9 @@ words and their occurency per day. the same Subscriber name in both of them. +This Module is also used for term frequency. - -zrank for each day -week -> top zrank for each day - +/!\ Top set management is done in the module Curve_manage_top_set Requirements @@ -117,48 +112,21 @@ if __name__ == "__main__": low_word = word.lower() - #Old curve + #Old curve with words in file r_serv1.hincrby(low_word, date, int(score)) - # Update redis curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score))) + + # Add in set only if term is not in the blacklist if low_word not in server_term.smembers(BlackListTermsSet_Name): server_term.zincrby(curr_set, low_word, float(score)) #Add more info for tracked terms check_if_tracked_term(low_word, filename) - # Manage Top set is done in module Curve_manage_top_sets - - ''' - if server_term.scard(curr_set) < top_term_freq_max_set_cardinality: - server_term.sadd(curr_set, low_word) - elif server_term.sismember(curr_set, low_word): - continue - - else: - - - #timer = time.clock() - curr_word_value = getValueOverRange(low_word, timestamp, curr_num_day) - #print 'curr_range', time.clock() - timer - top_termFreq = server_term.smembers(curr_set) - sorted_top_termFreq_set = [] - #timer = time.clock() - for word in top_termFreq: - word_value = getValueOverRange(word, timestamp, curr_num_day) - sorted_top_termFreq_set.append((word, word_value)) - - sorted_top_termFreq_set.sort(key=lambda tup: tup[1]) - #print 'whole_range', time.clock() - timer - - if curr_word_value > int(sorted_top_termFreq_set[0][1]): - print str(curr_num_day)+':', low_word, curr_word_value, '\t', sorted_top_termFreq_set[0][0], sorted_top_termFreq_set[0][1], '\t', curr_word_value > sorted_top_termFreq_set[0][1] - server_term.srem(curr_set, sorted_top_termFreq_set[0][0]) - server_term.sadd(curr_set, low_word) - ''' else: + if generate_new_graph: generate_new_graph = False print 'Building graph' diff --git a/bin/Curve_manage_top_sets.py b/bin/Curve_manage_top_sets.py index 92d63b33..34c1c238 100755 --- a/bin/Curve_manage_top_sets.py +++ b/bin/Curve_manage_top_sets.py @@ -2,10 +2,8 @@ # -*-coding:UTF-8 -* """ - - -zrank for each day -week -> top zrank for each day +This module manage top sets for terms frequency. +Every 'refresh_rate' update the weekly and monthly set Requirements @@ -22,13 +20,13 @@ import time import copy from pubsublogger import publisher from packages import lib_words -import os import datetime import calendar from Helper import Process # Config Variables +Refresh_rate = 60*5 #sec BlackListTermsSet_Name = "BlackListSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet" top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set @@ -91,7 +89,6 @@ def manage_top_set(): - if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger @@ -107,7 +104,6 @@ if __name__ == '__main__': port=p.config.get("Redis_Level_DB_TermFreq", "port"), db=p.config.get("Redis_Level_DB_TermFreq", "db")) - # FUNCTIONS # publisher.info("Script Curve_manage_top_set started") # Sent to the logging a description of the module @@ -116,15 +112,12 @@ if __name__ == '__main__': manage_top_set() while True: - # Get one message from the input queue + # Get one message from the input queue (module only work if linked with a queue) message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) print 'sleeping' - time.sleep(60) # sleep a long time then manage the set + time.sleep(Refresh_rate) # sleep a long time then manage the set manage_top_set() continue - # Do something with the message from the queue - #manage_top_set() - diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index b85c2ae5..973d15ae 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -1,7 +1,8 @@ #!/usr/bin/env python2 # -*-coding:UTF-8 -* """ - Template for new modules + This module makes statistics for some modules and providers + """ import time @@ -56,29 +57,6 @@ def compute_most_posted(server, message): server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword) print redis_progression_name_set -# if keyword in server.smembers(redis_progression_name_set): # if it is already in the set -# return -# -# if (server.scard(redis_progression_name_set) < max_set_cardinality): -# server.sadd(redis_progression_name_set, keyword) - -# else: #not in the set -# #Check value for all members -# member_set = [] -# for keyw in server.smembers(redis_progression_name_set): -# keyw_value = server.hget(paste_date, module+'-'+keyw) -# if keyw_value is not None: -# member_set.append((keyw, int(keyw_value))) -# else: #No data for this set for today -# member_set.append((keyw, int(0))) -# member_set.sort(key=lambda tup: tup[1]) -# if len(member_set) > 0: -# if member_set[0][1] < keyword_total_sum: -# #remove min from set and add the new one -# print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' -# server.srem(redis_progression_name_set, member_set[0][0]) -# server.sadd(redis_progression_name_set, keyword) - def compute_provider_info(server, path): redis_all_provider = 'all_provider_set' @@ -100,22 +78,6 @@ def compute_provider_info(server, path): new_avg = float(sum_size) / float(num_paste) server.hset(paste_provider +'_avg', paste_date, new_avg) - ''' - prev_num_paste = server.hget(paste_provider+'_num', paste_date) - if prev_num_paste is not None: - ok = server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1) - prev_sum_size = server.hget(paste_provider+'_size', paste_date) - - if prev_sum_size is not None: - ok = server.hset(paste_provider+'_size', paste_date, float(prev_sum_size)+paste_size) - new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1) - else: - ok = server.hset(paste_provider+'_size', paste_date, paste_size) - - else: - ok = server.hset(paste_provider+'_num', paste_date, 1) - prev_num_paste = 0 - ''' # # Compute Most Posted @@ -136,28 +98,6 @@ def compute_provider_info(server, path): server.zrem(redis_avg_size_name_set, member_set[0][0]) server.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) - ''' - if paste_provider not in server.smembers(redis_avg_size_name_set): # if it is already in the set - if (server.scard(redis_avg_size_name_set) < max_set_cardinality): - server.sadd(redis_avg_size_name_set, paste_provider) - - else: #set full capacity - #Check value for all members - member_set = [] - for provider in server.smembers(redis_avg_size_name_set): - curr_avg = 0.0 - curr_size = server.hget(provider+'_size', paste_date) - curr_num = server.hget(provider+'_num', paste_date) - if (curr_size is not None) and (curr_num is not None): - curr_avg = float(curr_size) / float(curr_num) - member_set.append((provider, curr_avg)) - member_set.sort(key=lambda tup: tup[1]) - if member_set[0][1] < new_avg: - #remove min from set and add the new one - print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' - server.srem(redis_avg_size_name_set, member_set[0][0]) - server.sadd(redis_avg_size_name_set, paste_provider) - ''' # Num # if set not full or provider already present @@ -172,27 +112,6 @@ def compute_provider_info(server, path): server.zrem(member_set[0][0]) server.zadd(redis_providers_name_set, float(num_paste), paste_provider) - ''' - if paste_provider not in server.smembers(redis_providers_name_set): # if it is already in the set - if (server.scard(redis_providers_name_set) < max_set_cardinality): - server.sadd(redis_providers_name_set, paste_provider) - - else: #set full capacity - #Check value for all members - member_set = [] - for provider in server.smembers(redis_providers_name_set): - curr_num = 0 - curr_num = server.hget(provider+'_num', paste_date) - if curr_num is not None: - member_set.append((provider, int(curr_num))) - member_set.sort(key=lambda tup: tup[1]) - if len(member_set) > 0: - if member_set[0][1] < int(prev_num_paste)+1: - #remove min from set and add the new one - print 'Num - adding ' +paste_provider+ '(' +str(int(prev_num_paste)+1)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' - server.srem(redis_providers_name_set, member_set[0][0]) - server.sadd(redis_providers_name_set, paste_provider) - ''' if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 83e2e033..97b1a9bb 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -173,6 +173,7 @@ def showpaste(content_range): if content_range != 0: p_content = p_content[0:content_range] + return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list) def getPastebyType(server, module_name): @@ -205,20 +206,6 @@ def get_top_relevant_data(server, module_name): member_set.insert(0, ("passed_days", days)) return member_set -# member_set = [] -# for keyw in server.smembers(redis_progression_name_set): -# redis_progression_name = module_name+'-'+keyw -# keyw_value = server.hget(date ,redis_progression_name) -# keyw_value = keyw_value if keyw_value is not None else 0 -# member_set.append((keyw, int(keyw_value))) -# member_set.sort(key=lambda tup: tup[1], reverse=True) -# if member_set[0][1] == 0: #No data for this date -# days += 1 -# continue -# else: -# member_set.insert(0, ("passed_days", days)) -# return member_set - def Term_getValueOverRange(word, startDate, num_day): passed_days = 0 @@ -357,34 +344,6 @@ def providersChart(): return jsonify(member_set) -''' - # Iterate over element in top_x_set and retreive their value - member_set = [] - for keyw in r_serv_charts.smembers(redis_provider_name_set): - redis_provider_name_size = keyw+'_'+'size' - redis_provider_name_num = keyw+'_'+'num' - keyw_value_size = r_serv_charts.hget(redis_provider_name_size, get_date_range(0)[0]) - keyw_value_size = keyw_value_size if keyw_value_size is not None else 0.0 - keyw_value_num = r_serv_charts.hget(redis_provider_name_num, get_date_range(0)[0]) - - if keyw_value_num is not None: - keyw_value_num = int(keyw_value_num) - else: - if module_name == "size": - keyw_value_num = 10000000000 - else: - keyw_value_num = 0 - if module_name == "size": - member_set.append((keyw, float(keyw_value_size)/float(keyw_value_num))) - else: - member_set.append((keyw, float(keyw_value_num))) - - member_set.sort(key=lambda tup: tup[1], reverse=True) - if len(member_set) == 0: - member_set.append(("No relevant data", float(100))) - return jsonify(member_set) -''' - @app.route("/search", methods=['POST']) def search(): @@ -771,50 +730,6 @@ def terms_plot_top_data(): return jsonify(to_return) - ''' - to_return = [] - for term in r_serv_term.smembers(the_set): - value_range = [] - tot_sum = 0 - for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): - value = r_serv_term.hget(timestamp, term) - curr_value_range = int(value) if value is not None else 0 - tot_sum += curr_value_range - value_range.append([timestamp, curr_value_range]) - - to_return.append([term, value_range, tot_sum]) - - return jsonify(to_return) - ''' - - -@app.route("/test/") #completely shows the paste in a new tab -def test(): - - server = r_serv_term - array1 = [] - for w in server.smembers('TopTermFreq_set_day'): - val = server.hget('1471564800', w) - val = val if val is not None else 0 - val2 = server.hget('1471478400', w) - val2 = val2 if val2 is not None else 0 - array1.append((w, (int(val), int(val2)))) - -# array2 = [] -# for w in server.smembers('TopTermFreq_set_week'): -# array2.append((w, int(server.hget('1471478400', w)))) - - array1.sort(key=lambda tup: tup[1][0]) - stri = "

day

" - for e in array1: - stri += "

"+ e[0] + "\t" + str(e[1]) +"

" -# stri += "

week

" -# for e in array2: -# stri += "

"+ e[0] + "\t" + str(e[1]) +"

" - - print stri - return stri - @app.route("/showsavedpaste/") #completely shows the paste in a new tab diff --git a/var/www/static/js/FlexGauge.js b/var/www/static/js/FlexGauge.js index e7db3bd9..3aa2bf7a 100644 --- a/var/www/static/js/FlexGauge.js +++ b/var/www/static/js/FlexGauge.js @@ -3,6 +3,8 @@ * Version: 1.0 * Author: Jeff Millies * Author URI: + * + * Slight modification for better display in Sentiment webpages */ (function ($) { var FlexGauge = function (o) { diff --git a/var/www/templates/important_paste_by_module.html b/var/www/templates/important_paste_by_module.html index fdb732ba..d4e6b5dd 100644 --- a/var/www/templates/important_paste_by_module.html +++ b/var/www/templates/important_paste_by_module.html @@ -115,7 +115,6 @@ $(document).ready(function(){ $('#myTable').on( 'draw.dt', function () { // On click, get html content from url and update the corresponding modal $("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) { - console.log('hi'); event.preventDefault(); var modal=$(this); var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num'); From 20fd191c20a76ec6772468be8c1a8f52f436c401 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 21:29:59 +0200 Subject: [PATCH 51/68] Suppressed prints --- bin/SentimentAnalyser.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/bin/SentimentAnalyser.py b/bin/SentimentAnalyser.py index 63f6c095..09f59e40 100755 --- a/bin/SentimentAnalyser.py +++ b/bin/SentimentAnalyser.py @@ -33,7 +33,6 @@ size_threshold = 250 line_max_length_threshold = 1000 def Analyse(message, server): - #print 'analyzing' path = message paste = Paste.Paste(path) @@ -50,19 +49,13 @@ def Analyse(message, server): if p_MimeType in accepted_Mime_type: - print 'Processing', path the_date = datetime.date(int(p_date[0:4]), int(p_date[4:6]), int(p_date[6:8])) - #print 'pastedate: ', the_date the_time = datetime.datetime.now() the_time = datetime.time(getattr(the_time, 'hour'), 0, 0) - #print 'now: ', the_time combined_datetime = datetime.datetime.combine(the_date, the_time) - #print 'combined: ', combined_datetime timestamp = calendar.timegm(combined_datetime.timetuple()) - #print 'timestamp: ', timestamp sentences = tokenize.sent_tokenize(p_content.decode('utf-8', 'ignore')) - #print len(sentences) if len(sentences) > 0: avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0} @@ -82,7 +75,6 @@ def Analyse(message, server): else: avg_score[k] += ss[k] - #print('{0}: {1}, '.format(k, ss[k])) for k in avg_score: if k == 'compoundPos': @@ -99,16 +91,13 @@ def Analyse(message, server): # (UniqID_i -> PasteValue_i) server.sadd('Provider_set', provider) - #print 'Provider_set', provider provider_timestamp = provider + '_' + str(timestamp) - #print provider_timestamp server.incr('UniqID') UniqID = server.get('UniqID') print provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines' server.sadd(provider_timestamp, UniqID) server.set(UniqID, avg_score) - #print UniqID, '->', avg_score else: print 'Dropped:', p_MimeType @@ -143,15 +132,12 @@ if __name__ == '__main__': port=p.config.get("Redis_Level_DB_Sentiment", "port"), db=p.config.get("Redis_Level_DB_Sentiment", "db")) - # Endless loop getting messages from the input queue while True: - # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue - # Do something with the message from the queue Analyse(message, server) From 8359a6668f3793a08079127029fb7364c5929882 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 22 Aug 2016 21:36:19 +0200 Subject: [PATCH 52/68] Suppressed useless comms --- var/www/Flask_server.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 97b1a9bb..29bf565b 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -199,7 +199,6 @@ def get_top_relevant_data(server, module_name): for date in get_date_range(15): redis_progression_name_set = 'top_'+ module_name +'_set_' + date member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True) - #print member_set if len(member_set) == 0: #No data for this date days += 1 else: @@ -214,8 +213,6 @@ def Term_getValueOverRange(word, startDate, num_day): curr_to_return = 0 for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): value = r_serv_term.hget(timestamp, word) - #print timestamp, word - #print value curr_to_return += int(value) if value is not None else 0 for i in num_day: if passed_days == i-1: @@ -322,9 +319,6 @@ def providersChart(): curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date) if module_name == "size": curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0) - #curr_value_num = curr_value_num if curr_value_num is not None else 0 - #curr_value_num = curr_value_num if int(curr_value_num) != 0 else 10000000000 - #curr_value = float(curr_value_size if curr_value_size is not None else 0.0) / float(curr_value_num) else: curr_value = float(curr_value_num if curr_value_num is not None else 0.0) From 85ab0e36fff423754900b8899fddc9a06b5b0302 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 07:11:03 +0200 Subject: [PATCH 53/68] In sentiment analysis. if today top provider set is empty, try to take yesterday data and then takes from all_providers. --- var/www/Flask_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 29bf565b..6c01c689 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -463,6 +463,10 @@ def sentiment_analysis_getplotdata(): to_return = {} range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) + # if empty, get yesterday top providers + print 'providers_set_'+ get_date_range(1)[1] + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers + # if still empty, takes from all providers if range_providers == []: print 'today provider empty' range_providers = r_serv_charts.smembers('all_provider_set') From 6be557890c2d97ffd48a84461e1d06a611912a65 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 07:19:51 +0200 Subject: [PATCH 54/68] Replaced negative hour by 24h based. --- var/www/static/js/sentiment_trending.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index 2caf91d4..a56723b8 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -1,8 +1,11 @@ function generate_offset_to_time(num){ var to_ret = {}; - for(i=0; i<=num; i++) - to_ret[i] = new Date().getHours()-(23-i)+'h'; + for(i=0; i<=num; i++) { + var t1 = new Date().getHours()-(23-i); + t1 = t1 < 0 ? 24+t1 : t1; + to_ret[i] = t1+'h'; + } return to_ret; }; @@ -13,7 +16,9 @@ for(i=day; i>=0; i--){ for(j=0; j<24; j++){ var t1 =now.getDate()-i + ":"; - var t2 =now.getHours()-(23-j)+"h"; + var t2 =now.getHours()-(23-j); + t2 = t2 < 0 ? 24+t2 : t2; + t2 += "h"; to_ret[j+24*(day-i)] = t1+t2; } } From 047331df3e5c29637027d2af4df3b503c782dfc4 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 08:52:54 +0200 Subject: [PATCH 55/68] Added save of paste_baseName in set for searches. This is done in ModuleStats to not surcharge the Global module. --- bin/ModuleStats.py | 45 ++++++++++++++++++++++++----------------- var/www/Flask_server.py | 30 ++++++++++++++++----------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index 973d15ae..648649f7 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -58,11 +58,12 @@ def compute_most_posted(server, message): print redis_progression_name_set -def compute_provider_info(server, path): +def compute_provider_info(server_trend, server_pasteName, path): redis_all_provider = 'all_provider_set' paste = Paste.Paste(path) + paste_baseName = paste.p_name.split('.')[0] paste_size = paste._get_p_size() paste_provider = paste.p_source paste_date = str(paste._get_p_date()) @@ -71,12 +72,13 @@ def compute_provider_info(server, path): redis_providers_name_set = 'providers_set_' + paste_date # Add/Update in Redis - server.sadd(redis_all_provider, paste_provider) + server_pasteName.sadd(paste_baseName, path) + server_trend.sadd(redis_all_provider, paste_provider) - num_paste = int(server.hincrby(paste_provider+'_num', paste_date, 1)) - sum_size = float(server.hincrbyfloat(paste_provider+'_size', paste_date, paste_size)) + num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1)) + sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size)) new_avg = float(sum_size) / float(num_paste) - server.hset(paste_provider +'_avg', paste_date, new_avg) + server_trend.hset(paste_provider +'_avg', paste_date, new_avg) # @@ -84,33 +86,33 @@ def compute_provider_info(server, path): # # Size - if server.zcard(redis_sum_size_set) < max_set_cardinality or server.zscore(redis_sum_size_set, paste_provider) != "nil": - server.zadd(redis_sum_size_set, float(num_paste), paste_provider) - server.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) + if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil": + server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider) + server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) else: #set full capacity - member_set = server.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1) + member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1) # Member set is a list of (value, score) pairs if float(member_set[0][1]) < new_avg: #remove min from set and add the new one print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' - server.zrem(redis_sum_size_set, member_set[0][0]) - server.zadd(redis_sum_size_set, float(sum_size), paste_provider) - server.zrem(redis_avg_size_name_set, member_set[0][0]) - server.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) + server_trend.zrem(redis_sum_size_set, member_set[0][0]) + server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider) + server_trend.zrem(redis_avg_size_name_set, member_set[0][0]) + server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) # Num # if set not full or provider already present - if server.zcard(redis_providers_name_set) < max_set_cardinality or server.zscore(redis_providers_name_set, paste_provider) != "nil": - server.zadd(redis_providers_name_set, float(num_paste), paste_provider) + if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil": + server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider) else: #set at full capacity - member_set = server.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1) + member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1) # Member set is a list of (value, score) pairs if int(member_set[0][1]) < num_paste: #remove min from set and add the new one print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' - server.zrem(member_set[0][0]) - server.zadd(redis_providers_name_set, float(num_paste), paste_provider) + server_trend.zrem(member_set[0][0]) + server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider) if __name__ == '__main__': @@ -135,6 +137,11 @@ if __name__ == '__main__': port=p.config.get("Redis_Level_DB_Trending", "port"), db=p.config.get("Redis_Level_DB_Trending", "db")) + r_serv_pasteName = redis.StrictRedis( + host=p.config.get("Redis_Paste_Name", "host"), + port=p.config.get("Redis_Paste_Name", "port"), + db=p.config.get("Redis_Paste_Name", "db")) + # Endless loop getting messages from the input queue while True: # Get one message from the input queue @@ -151,4 +158,4 @@ if __name__ == '__main__': if len(message.split(';')) > 1: compute_most_posted(r_serv_trend, message) else: - compute_provider_info(r_serv_trend, message) + compute_provider_info(r_serv_trend, r_serv_pasteName, message) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 6c01c689..90267949 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -61,6 +61,11 @@ r_serv_term = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_TermFreq", "port"), db=cfg.getint("Redis_Level_DB_TermFreq", "db")) +r_serv_pasteName = redis.StrictRedis( + host=cfg.get("Redis_Paste_Name", "host"), + port=cfg.getint("Redis_Paste_Name", "port"), + db=cfg.getint("Redis_Paste_Name", "db")) + app = Flask(__name__, static_url_path='/static/') @@ -348,19 +353,20 @@ def search(): c = [] #preview of the paste content paste_date = [] paste_size = [] + # Search filename - from os import walk - for (dirpath, dirnames, filenames) in walk(os.path.join(os.environ['AIL_HOME'], 'PASTES/')): - if q[0] in filenames: - r.append(dirpath+'/'+q[0]) - paste = Paste.Paste(dirpath+'/'+q[0]) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - c.append(content[0:content_range]) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - paste_date.append(curr_date) - paste_size.append(paste._get_p_size()) + print r_serv_pasteName.smembers(q[0]) + for path in r_serv_pasteName.smembers(q[0]): + print path + r.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + c.append(content[0:content_range]) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_size.append(paste._get_p_size()) # Search full line from whoosh import index From 1766ced9e66fdb150f6c088d0509b7544c704a32 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 09:08:17 +0200 Subject: [PATCH 56/68] Fixed potential XSS problem with pastes display in 'search' and 'important_paste_by_module'. --- var/www/templates/important_paste_by_module.html | 3 ++- var/www/templates/search.html | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/var/www/templates/important_paste_by_module.html b/var/www/templates/important_paste_by_module.html index d4e6b5dd..061648c4 100644 --- a/var/www/templates/important_paste_by_module.html +++ b/var/www/templates/important_paste_by_module.html @@ -92,7 +92,8 @@ $(document).ready(function(){ } if (final_index != start_index){ // still have data to display - $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1)); // Append the new content + // Append the new content using text() and not append (XSS) + $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1)); start_index = final_index; if (flag_stop) nothing_to_display(); diff --git a/var/www/templates/search.html b/var/www/templates/search.html index 43459b1f..5ef83daa 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -196,7 +196,8 @@ } if (final_index != start_index){ // still have data to display - $("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content + // Append the new content using text() and not append (XSS) + $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text() + complete_paste.substring(start_index+1, final_index+1)); start_index = final_index; if (flag_stop) nothing_to_display(); From f9b27e401353153e7ce0c55c2fecd77579e0d213 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 09:50:03 +0200 Subject: [PATCH 57/68] Fixed bug in curve preventing to track concerned_pastes. Now all tracked words are put in lower case. --- bin/Curve.py | 3 ++- var/www/Flask_server.py | 8 ++++---- var/www/templates/terms_management.html | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/Curve.py b/bin/Curve.py index 9337bc5b..59557881 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -47,10 +47,11 @@ top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, to def check_if_tracked_term(term, path): - if term in TrackedTermsSet_Name: + if term in server_term.smembers(TrackedTermsSet_Name): #add_paste to tracked_word_set set_name = "tracked_" + term server_term.sadd(set_name, path) + print term, 'addded', set_name, '->', path p.populate_set_out("New Term added", 'CurveManageTopSets') diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 90267949..07878927 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -649,16 +649,16 @@ def terms_management_action(): else: if section == "followTerm": if action == "add": - r_serv_term.sadd(TrackedTermsSet_Name, term) + r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp) else: - r_serv_term.srem(TrackedTermsSet_Name, term) + r_serv_term.srem(TrackedTermsSet_Name, term.lower()) elif section == "blacklistTerm": if action == "add": - r_serv_term.sadd(BlackListTermsSet_Name, term) + r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) else: - r_serv_term.srem(BlackListTermsSet_Name, term) + r_serv_term.srem(BlackListTermsSet_Name, term.lower()) else: return "None" diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index d070715c..487df3a7 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -289,9 +289,9 @@ function perform_operation(){ var curr_section = $(this).attr('data-section'); var curr_action = $(this).attr('data-action'); if (curr_action == "add") { - var curr_term = $('#'+curr_section+'Input').val(); + var curr_term = $('#'+curr_section+'Input').val().toLowerCase();; } else { - var curr_term = $(this).attr('data-content'); + var curr_term = $(this).attr('data-content').toLowerCase();; } var data_to_send = { section: curr_section, action:curr_action, term: curr_term}; From 697812dbff18d9d3bd0e4067f7ea1918ffb0a7a6 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 09:59:39 +0200 Subject: [PATCH 58/68] Added config.sample --- bin/packages/config.cfg.sample | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 1406638d..e74b5da2 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -1,6 +1,5 @@ [Directories] bloomfilters = Blooms -#Duplicate_ssdeep dicofilters = Dicos pastes = PASTES @@ -59,11 +58,26 @@ host = localhost port = 6379 db = 1 +[Redis_Paste_Name] +host = localhost +port = 6379 +db = 2 + ##### LevelDB ##### [Redis_Level_DB_Curve] host = localhost -port = 3016 -db = 0 +port = 6382 +db = 1 + +[Redis_Level_DB_Sentiment] +host = localhost +port = 6382 +db = 4 + +[Redis_Level_DB_TermFreq] +host = localhost +port = 6382 +db = 2 [Redis_Level_DB] host = localhost @@ -72,8 +86,8 @@ db = 0 [Redis_Level_DB_Trending] host = localhost -port = 2016 -db = 0 +port = 6382 +db = 3 [Redis_Level_DB_Hashs] host = localhost From 32dfd2b18196da08d9353351d6d027358cb86dcd Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 10:16:33 +0200 Subject: [PATCH 59/68] Moved legend place in term-plot-top --- var/www/templates/terms_plot_top.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index da3611bf..cee47f06 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -293,17 +293,22 @@ var graph_options = { tickColor: "#f9f9f9", borderWidth: 0 }, + legend: { show: true, + noColumns: 0, + position: "nw" + }, xaxis: { mode: "time", timeformat: "%m/%d", minTickSize: [1, "day"] }, yaxis: { + //transform: function (v) { return v < 1 ? v : Math.log(v); } autoscaleMargin: 0.1, }, tooltip: true, tooltipOpts: { - content: "%s (%x.1 is %y.4)", + content: " %s (%x.1 is %y.4) ", shifts: { x: -60, y: 25 @@ -323,7 +328,6 @@ var plot_month; var promises = []; // Used to know when everything has been received promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today, num_day: 5 }, function(data, status){ - console.log(data); data.sort(function(a, b){return b[2]-a[2];}); // Sort data var table_today = $("#table-today") @@ -335,7 +339,6 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]); } to_plot.push({ data: curr_data, label: data[i][0]}); - console.log(to_plot); if ( i < (data.length/2)) table_today.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0])+"") else From 8c956c22b17299611515690060597e3493d060df Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 11:02:36 +0200 Subject: [PATCH 60/68] Added autohide curves in terms-top-set. --- var/www/templates/terms_plot_top.html | 47 ++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index cee47f06..c7941b48 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -320,6 +320,7 @@ var graph_options = { set_today = "TopTermFreq_set_day"; set_week = "TopTermFreq_set_week"; set_month = "TopTermFreq_set_month"; +default_num_curves = 8; var plot_today; var plot_week; @@ -333,18 +334,23 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today var table_today = $("#table-today") var table_today2 = $("#table-today2") var to_plot = []; + var unchecked_label = []; for(i=0; i=default_num_curves) { + unchecked_label.push(data[i][0]); + } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table_today.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0])+"") + table_today.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0], i") else - table_today2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0])+"") + table_today2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("today", data[i][0], i") } plot_today = $.plot($("#graph-today"), to_plot, graph_options); + hide_unchecked_curves(plot_today, unchecked_label); $("#graph-today").bind("plothover", function (event, pos, item) { if (item) { var date = new Date(item.datapoint[0]); @@ -367,18 +373,23 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, var table = $("#table-week") var table2 = $("#table-week2") var to_plot = []; + var unchecked_label = []; for(i=0; i=default_num_curves) { + unchecked_label.push(data[i][0]); + } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0])+"") + table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0], i") else - table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0])+"") + table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("week", data[i][0], i") } plot_week = $.plot($("#graph-week"), to_plot, graph_options); + hide_unchecked_curves(plot_week, unchecked_label); $("#graph-week").bind("plothover", function (event, pos, item) { if (item) { var date = new Date(item.datapoint[0]); @@ -400,18 +411,23 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month var table = $("#table-month") var table2 = $("#table-month2") var to_plot = []; + var unchecked_label = []; for(i=0; i=default_num_curves) { + unchecked_label.push(data[i][0]); + } to_plot.push({ data: curr_data, label: data[i][0]}); if ( i < (data.length/2)) - table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0])+"") + table.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0], i") else - table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0])+"") + table2.append(""+data[i][0]+""+data[i][2]+""+addbuttons(data[i][0])+""+addcheckbox("month", data[i][0], i") } plot_month = $.plot($("#graph-month"), to_plot, graph_options); + hide_unchecked_curves(plot_month, unchecked_label); $("#graph-month").bind("plothover", function (event, pos, item) { if (item) { var date = new Date(item.datapoint[0]); @@ -445,8 +461,9 @@ function addbuttons(term) { "data-section=\"blacklistTerm\" data-term=\""+term+"\">"; } -function addcheckbox(graph, term) { - return ""; +function addcheckbox(graph, term, checked) { + var checked_text = checked ? "checked" : ""; + return ""; } function perform_operation(){ @@ -462,6 +479,20 @@ function perform_operation(){ }); } + +function hide_unchecked_curves(plot, unchecked_label) { + var graphData = plot.getData(); + var index; + for(i=0; i Date: Tue, 23 Aug 2016 11:20:48 +0200 Subject: [PATCH 61/68] harmonized tooltip and popover display --- var/www/templates/search.html | 3 +-- var/www/templates/terms_management.html | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/var/www/templates/search.html b/var/www/templates/search.html index 5ef83daa..2616a145 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -23,12 +23,11 @@ .tooltip-inner { text-align: left; height: 200%; - width: 200%; max-width: 500px; max-height: 500px; font-size: 13px; } - xmp { + pre { white-space:pre-wrap; word-wrap:break-word; } diff --git a/var/www/templates/terms_management.html b/var/www/templates/terms_management.html index 487df3a7..3093837f 100644 --- a/var/www/templates/terms_management.html +++ b/var/www/templates/terms_management.html @@ -26,9 +26,15 @@ padding-left:0; list-style:none } + .btn-link { color: #000000 } + + .popover-content { + white-space:pre-wrap; + word-wrap:break-word; + } From 0eba419651857f2a437c5dc588221cd1e8b99487 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 11:56:32 +0200 Subject: [PATCH 62/68] Added multiple terms plot in terms-plot-tool. --- var/www/templates/terms_plot_tool.html | 37 +++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index 7e20fe7a..def0e9d2 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -77,7 +77,8 @@
- + +
@@ -143,6 +144,7 @@ " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() ); $('#plot-btn').click(plotData); + $('#plot-btn-add').click(addData); $("#TermInput").val($("#TermInput").attr("data-init-plot")); $("#plot-btn").click(); @@ -158,6 +160,8 @@ From e59b7c4e47ec0ad932b87e57770bc94f50175bcb Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 12:03:23 +0200 Subject: [PATCH 63/68] Added tooltip in graph for terms-plot-tool --- var/www/templates/terms_plot_tool.html | 18 ++++++++++++++++++ var/www/templates/terms_plot_top.html | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/var/www/templates/terms_plot_tool.html b/var/www/templates/terms_plot_tool.html index def0e9d2..6108eb91 100644 --- a/var/www/templates/terms_plot_tool.html +++ b/var/www/templates/terms_plot_tool.html @@ -119,6 +119,9 @@ + + + From 42b3966499390043e21dea1d4ed91f64b381dc44 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 16:01:11 +0200 Subject: [PATCH 67/68] Added new module in individual launch scripts --- bin/launch_redis.sh | 3 +++ bin/launch_scripts.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh index ffdbdb58..c7af27c8 100755 --- a/bin/launch_redis.sh +++ b/bin/launch_redis.sh @@ -14,3 +14,6 @@ screen -S "Redis" -X screen -t "6380" bash -c '../redis/src/redis-server '$conf_ sleep 0.1 screen -S "Redis" -X screen -t "6381" bash -c '../redis/src/redis-server '$conf_dir'6381.conf ; read x' +# For Words and curves +sleep 0.1 +screen -S "Redis" -X screen -t "6382" bash -c '../redis/src/redis-server '$conf_dir'6382.conf ; read x' diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index 4b1abc5f..ad55244a 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -34,6 +34,8 @@ screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 +screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x' +sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' From e3daa841963123b61a4e6762ad688c5bd706be2b Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 23 Aug 2016 16:09:34 +0200 Subject: [PATCH 68/68] Added icon in header --- var/www/templates/header.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/var/www/templates/header.html b/var/www/templates/header.html index 4deb4740..d2bfe0a6 100644 --- a/var/www/templates/header.html +++ b/var/www/templates/header.html @@ -7,16 +7,16 @@
  • Sentiment Analysis
  • Terms frequency