AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-30 01:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added draft of filter in sentiment analysis (Discard syntaxical languages) + Added nice tooltip for sparkline. Trending displays avg in function of the number of elements processed and not for the complete week + fixed bug in gauge and canvasjs (was performing avg with only 1 graph instead of all 8).

Browse source
This commit is contained in:
Mokaddem 2016-08-16 16:33:02 +02:00
parent ecd834ffb6
commit 1084e45f1b
4 changed files with 156 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
bin/SentimentAnalyser.py
View file

@ -15,6 +15,7 @@ import time
import datetime
import calendar
import redis
import json
from pubsublogger import publisher
from Helper import Process
from packages import Paste
@ -22,6 +23,8 @@ from packages import Paste
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk import tokenize
# Config Variables
accepted_Mime_type = ['text/plain']
def Analyse(message, server):
#print 'analyzing'
@ -31,8 +34,15 @@ def Analyse(message, server):
content = paste.get_p_content()
provider = paste.p_source
p_date = str(paste._get_p_date())
#print provider, date
p_MimeType = paste._get_p_encoding()
# Perform further analysis
if p_MimeType == "text/plain":
if isJSON(content):
p_MimeType = "JSON"
if p_MimeType in accepted_Mime_type:
print 'Processing', path
the_date = datetime.date(int(p_date[0:4]), int(p_date[4:6]), int(p_date[6:8]))
#print 'pastedate: ', the_date
the_time = datetime.datetime.now()
@ -89,10 +99,19 @@ def Analyse(message, server):
print provider_timestamp, '->', UniqID
server.sadd(provider_timestamp, UniqID)
server.set(UniqID, avg_score)
print avg_score
#print UniqID, '->', avg_score
else:
print 'Dropped:', p_MimeType
#print '(', provider, timestamp, str(avg_score) , ')'
#server.hset(provider, timestamp, str(avg_score))
def isJSON(content):
try:
json.loads(content)
return True
except Exception,e:
return False
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@ -102,7 +121,7 @@ if __name__ == '__main__':
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'SentimentAnalyser'
config_section = 'SentimentAnalysis'
# Setup the I/O queues
p = Process(config_section)

4
bin/packages/modules.cfg
View file

@ -66,8 +66,8 @@ subscribe = Redis_BrowseWarningPaste
#subscribe = Redis_Cve
#publish = Redis_BrowseWarningPaste
[SentimentAnalyser]
subscribe = Redis_LinesLong
[SentimentAnalysis]
subscribe = Redis_Global
[Release]
subscribe = Redis_Global

2
var/www/Flask_server.py
View file

@ -497,7 +497,7 @@ def sentiment_analysis_plot_tool_getdata():
timestamp1 = calendar.timegm(date1.timetuple())
timestamp2 = calendar.timegm(date2.timetuple())
print timestamp2
oneHour = 60*60
oneDay = oneHour*24

66
var/www/static/js/sentiment_trending.js
View file

@ -1,4 +1,26 @@
function generate_offset_to_time(num){
var to_ret = {};
for(i=0; i<=num; i++)
to_ret[i] = new Date().getHours()-(23-i)+'h';
return to_ret;
};
function generate_offset_to_date(day){
var now = new Date();
var to_ret = {};
for(i=0; i<day; i++){
for(j=0; j<24; j++){
var t1 =now.getDate()-i + ":";
var t2 =now.getHours()-(23-j)+"h";
to_ret[j+24*i] = t1+t2;
}
}
return to_ret;
};
var offset_to_time = generate_offset_to_time(23);
var offset_to_date = generate_offset_to_date(7);
var sparklineOptions = {
height: 80,//Height of the chart - Defaults to 'auto' (line height of the containing tag)
@ -13,6 +35,7 @@
negBarColor: '#f22929',
zeroColor: '#ffff00',
tooltipFormat: '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>',
};
@ -37,7 +60,9 @@ $.getJSON("/sentiment_analysis_getplotdata/",
var spark_data = [];
var curr_provider = array_provider[graphNum];
var curr_sum = 0.0;
var curr_sum_elem = 0.0;
var day_sum = 0.0;
var day_sum_elem = 0.0;
var hour_sum = 0.0;
for(curr_date=dateStart; curr_date<dateStart+oneWeek; curr_date+=oneHour){
@ -71,10 +96,12 @@ $.getJSON("/sentiment_analysis_getplotdata/",
graph_data.push({'neg': neg, 'neu': neu, 'pos': pos, 'compoundPos': compPosAvg, 'compoundNeg': compNegAvg});
spark_data.push(pos-neg);
curr_sum += (pos-neg);
curr_sum_elem++;
max_value = Math.abs(pos-neg) > max_value ? Math.abs(pos-neg) : max_value;
if(curr_date >= dateStart+oneWeek-24*oneHour){
day_sum += (pos-neg);
day_sum_elem++;
}
if(curr_date >= dateStart+oneWeek-oneHour){
hour_sum += (pos-neg);
@ -85,7 +112,8 @@ $.getJSON("/sentiment_analysis_getplotdata/",
all_graph_day_sum += day_sum;
all_graph_hour_sum += hour_sum;
var curr_avg = curr_sum / (oneWeek/oneHour);
var curr_avg = curr_sum / (curr_sum_elem);
//var curr_avg = curr_sum / (oneWeek/oneHour);
//var curr_avg = curr_sum / (spark_data.length);
graph_avg.push([curr_provider, curr_avg]);
plot_data.push(spark_data);
@ -94,6 +122,8 @@ $.getJSON("/sentiment_analysis_getplotdata/",
sparklineOptions.chartRangeMax = max_value;
sparklineOptions.chartRangeMin = -max_value;
sparklineOptions.tooltipValueLookups = { names: offset_to_date};
// print week
var num = graphNum + 1;
var placeholder = '.sparkLineStatsWeek' + num;
@ -102,12 +132,15 @@ $.getJSON("/sentiment_analysis_getplotdata/",
$(placeholder+'s').text(curr_avg.toFixed(5));
sparklineOptions.barWidth = 18;
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
$(placeholder+'b').sparkline([curr_avg], sparklineOptions);
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
sparklineOptions.barWidth = 2;
sparklineOptions.tooltipValueLookups = { names: offset_to_time};
// print today
var data_length = plot_data[graphNum].length;
var data_today = plot_data[graphNum].slice(data_length-24, data_length-1);
var data_today = plot_data[graphNum].slice(data_length-24, data_length);
placeholder = '.sparkLineStatsToday' + num;
sparklineOptions.barWidth = 14;
@ -115,9 +148,13 @@ $.getJSON("/sentiment_analysis_getplotdata/",
$(placeholder+'t').text(curr_provider);
sparklineOptions.barWidth = 18;
$(placeholder+'b').sparkline([day_sum/24], sparklineOptions);
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
//var day_avg = day_sum/24;
var day_avg = day_sum/day_sum_elem;
$(placeholder+'b').sparkline([day_avg], sparklineOptions);
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
sparklineOptions.barWidth = 2;
$(placeholder+'s').text((day_sum/24).toFixed(5));
$(placeholder+'s').text((day_avg).toFixed(5));
}//for loop
@ -153,13 +190,15 @@ $.getJSON("/sentiment_analysis_getplotdata/",
gaugeOptions.appendTo = '#gauge_today_last_hour';
gaugeOptions.dialLabel = 'Last hour';
gaugeOptions.elementId = 'gauge1';
gaugeOptions.inc = all_graph_hour_sum / 8;
var piePercent = (all_graph_hour_sum / 8) / max_value;
gaugeOptions.inc = piePercent;
var gauge_today_last_hour = new FlexGauge(gaugeOptions);
gaugeOptions2.appendTo = '#gauge_today_last_days';
gaugeOptions2.dialLabel = 'Today';
gaugeOptions2.elementId = 'gauge2';
gaugeOptions2.inc = all_graph_day_sum / 8;
piePercent = (all_graph_day_sum / (8*24)) / max_value;
gaugeOptions2.inc = piePercent;
var gauge_today_last_days = new FlexGauge(gaugeOptions2);
gaugeOptions3.appendTo = '#gauge_week';
@ -167,10 +206,14 @@ $.getJSON("/sentiment_analysis_getplotdata/",
gaugeOptions3.elementId = 'gauge3';
var graph_avg_sum = 0.0;
for (i=0; i<graph_avg.length; i++)
var temp_max_val = 0.0;
for (i=0; i<graph_avg.length; i++){
graph_avg_sum += graph_avg[i][1];
temp_max_val = Math.abs(graph_avg[i][1]) > temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val;
}
gaugeOptions3.inc = graph_avg_sum / graph_avg.length;
piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val;
gaugeOptions3.inc = piePercent;
var gauge_today_last_days = new FlexGauge(gaugeOptions3);
@ -185,21 +228,24 @@ $.getJSON("/sentiment_analysis_getplotdata/",
/* ----------- CanvasJS ------------ */
var gauge_data = graph_data.slice(graph_data.length-24*2, graph_data.length-24*1);
var comp_sum_day_pos = 0.0;
var comp_sum_day_neg = 0.0;
var comp_sum_hour_pos = 0.0;
var comp_sum_hour_neg = 0.0;
for(graphNum=0; graphNum<8; graphNum++){
curr_graphData = all_data[graphNum];
var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length);
for (i=1; i< gauge_data.length; i++){
comp_sum_day_pos += gauge_data[i].compoundPos;
comp_sum_day_neg += gauge_data[i].compoundNeg;
if(i >= 24){
if(i == 23){
comp_sum_hour_pos += gauge_data[i].compoundPos;
comp_sum_hour_neg += gauge_data[i].compoundNeg;
}
}
}
var options_canvasJS_1 = {