Merge pull request #68 from mokaddem/multiple-modif

Multiple modif
This commit is contained in:
Raphaël Vinot 2016-08-23 16:17:16 +02:00 committed by GitHub
commit fed9aeab44
32 changed files with 3749 additions and 196 deletions

View file

@ -104,6 +104,7 @@ LICENSE
Copyright (C) 2014-2016 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) Copyright (C) 2014-2016 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
Copyright (c) 2014-2016 Raphaël Vinot Copyright (c) 2014-2016 Raphaël Vinot
Copyright (c) 2014-2016 Alexandre Dulaunoy Copyright (c) 2014-2016 Alexandre Dulaunoy
Copyright (c) 2016 Sami Mokaddem
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by it under the terms of the GNU Affero General Public License as published by

View file

@ -66,13 +66,12 @@ if __name__ == "__main__":
publisher.warning('{}Checked {} valid number(s)'.format( publisher.warning('{}Checked {} valid number(s)'.format(
to_print, len(creditcard_set))) to_print, len(creditcard_set)))
#Send to duplicate #Send to duplicate
p.populate_set_out(filepath, 'Redis_Duplicate') p.populate_set_out(filepath, 'Duplicate')
#send to Browse_warning_paste #send to Browse_warning_paste
p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
else: else:
publisher.info('{}CreditCard related'.format(to_print)) publisher.info('{}CreditCard related'.format(to_print))
else: else:
publisher.debug("Script creditcard is idling 1m") publisher.debug("Script creditcard is idling 1m")
print 'Sleeping'
time.sleep(10) time.sleep(10)

View file

@ -1,9 +1,6 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_Sub_Curve Module
============================
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module. This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
This modules update a .csv file used to draw curves representing selected This modules update a .csv file used to draw curves representing selected
@ -14,6 +11,12 @@ words and their occurency per day.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put ..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them. the same Subscriber name in both of them.
This Module is also used for term frequency.
/!\ Top set management is done in the module Curve_manage_top_set
Requirements Requirements
------------ ------------
@ -28,9 +31,39 @@ from pubsublogger import publisher
from packages import lib_words from packages import lib_words
import os import os
import datetime import datetime
import calendar
from Helper import Process from Helper import Process
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def check_if_tracked_term(term, path):
if term in server_term.smembers(TrackedTermsSet_Name):
#add_paste to tracked_word_set
set_name = "tracked_" + term
server_term.sadd(set_name, path)
print term, 'addded', set_name, '->', path
p.populate_set_out("New Term added", 'CurveManageTopSets')
def getValueOverRange(word, startDate, num_day):
to_return = 0
for timestamp in range(startDate, startDate - num_day*oneDay, -oneDay):
value = server_term.hget(timestamp, word)
to_return += int(value) if value is not None else 0
return to_return
if __name__ == "__main__": if __name__ == "__main__":
publisher.port = 6380 publisher.port = 6380
publisher.channel = "Script" publisher.channel = "Script"
@ -44,6 +77,11 @@ if __name__ == "__main__":
port=p.config.get("Redis_Level_DB_Curve", "port"), port=p.config.get("Redis_Level_DB_Curve", "port"),
db=p.config.get("Redis_Level_DB_Curve", "db")) db=p.config.get("Redis_Level_DB_Curve", "db"))
server_term = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
# FUNCTIONS # # FUNCTIONS #
publisher.info("Script Curve started") publisher.info("Script Curve started")
@ -56,22 +94,40 @@ if __name__ == "__main__":
message = p.get_from_set() message = p.get_from_set()
prec_filename = None prec_filename = None
generate_new_graph = False generate_new_graph = False
# Term Frequency
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
while True: while True:
if message is not None: if message is not None:
generate_new_graph = True generate_new_graph = True
filename, word, score = message.split() filename, word, score = message.split()
temp = filename.split('/') temp = filename.split('/')
date = temp[-4] + temp[-3] + temp[-2] date = temp[-4] + temp[-3] + temp[-2]
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
curr_set = top_termFreq_setName_day[0] + str(timestamp)
low_word = word.lower() low_word = word.lower()
prev_score = r_serv1.hget(low_word, date) #Old curve with words in file
if prev_score is not None: r_serv1.hincrby(low_word, date, int(score))
r_serv1.hset(low_word, date, int(prev_score) + int(score))
else: # Update redis
r_serv1.hset(low_word, date, score) curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score)))
# Add in set only if term is not in the blacklist
if low_word not in server_term.smembers(BlackListTermsSet_Name):
server_term.zincrby(curr_set, low_word, float(score))
#Add more info for tracked terms
check_if_tracked_term(low_word, filename)
else: else:
if generate_new_graph: if generate_new_graph:
generate_new_graph = False generate_new_graph = False
print 'Building graph' print 'Building graph'

123
bin/Curve_manage_top_sets.py Executable file
View file

@ -0,0 +1,123 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
This module manage top sets for terms frequency.
Every 'refresh_rate' update the weekly and monthly set
Requirements
------------
*Need running Redis instances. (Redis)
*Categories files of words in /files/ need to be created
*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
"""
import redis
import time
import copy
from pubsublogger import publisher
from packages import lib_words
import datetime
import calendar
from Helper import Process
# Config Variables
Refresh_rate = 60*5 #sec
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
num_day_month = 31
num_day_week = 7
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def manage_top_set():
startDate = datetime.datetime.now()
startDate = startDate.replace(hour=0, minute=0, second=0, microsecond=0)
startDate = calendar.timegm(startDate.timetuple())
dico = {}
# Retreive top data (2*max_card) from days sets
for timestamp in range(startDate, startDate - top_termFreq_setName_month[1]*oneDay, -oneDay):
curr_set = top_termFreq_setName_day[0] + str(timestamp)
array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2)
for word, value in array_top_day:
if word not in server_term.smembers(BlackListTermsSet_Name):
if word in dico.keys():
dico[word] += value
else:
dico[word] = value
if timestamp == startDate - num_day_week*oneDay:
dico_week = copy.deepcopy(dico)
# convert dico into sorted array
array_month = []
for w, v in dico.iteritems():
array_month.append((w, v))
array_month.sort(key=lambda tup: -tup[1])
array_month = array_month[0:20]
array_week = []
for w, v in dico_week.iteritems():
array_week.append((w, v))
array_week.sort(key=lambda tup: -tup[1])
array_week = array_week[0:20]
# suppress every terms in top sets
for curr_set, curr_num_day in top_termFreq_set_array[1:3]:
for w in server_term.zrange(curr_set, 0, -1):
server_term.zrem(curr_set, w)
# Add top term from sorted array in their respective sorted sets
for elem in array_week:
server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0])
for elem in array_month:
server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0])
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
config_section = 'CurveManageTopSets'
p = Process(config_section)
server_term = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
publisher.info("Script Curve_manage_top_set started")
# Sent to the logging a description of the module
publisher.info("Manage the top sets with the data created by the module curve.")
manage_top_set()
while True:
# Get one message from the input queue (module only work if linked with a queue)
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
print 'sleeping'
time.sleep(Refresh_rate) # sleep a long time then manage the set
manage_top_set()
continue

View file

@ -58,6 +58,10 @@ function launching_redis {
screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
sleep 0.1 sleep 0.1
screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
# For Words and curves
sleep 0.1
screen -S "Redis" -X screen -t "6382" bash -c 'redis-server '$conf_dir'6382.conf ; read x'
} }
function launching_lvldb { function launching_lvldb {
@ -134,6 +138,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
@ -151,6 +157,9 @@ function launching_scripts {
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x'
} }
#If no params, display the help #If no params, display the help

View file

@ -1,7 +1,8 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
Template for new modules This module makes statistics for some modules and providers
""" """
import time import time
@ -15,7 +16,7 @@ from Helper import Process
from packages import Paste from packages import Paste
# Config Var # Config Var
max_set_cardinality = 7 max_set_cardinality = 8
def get_date_range(num_day): def get_date_range(num_day):
curr_date = datetime.date.today() curr_date = datetime.date.today()
@ -30,14 +31,10 @@ def get_date_range(num_day):
def compute_most_posted(server, message): def compute_most_posted(server, message):
module, num, keyword, paste_date = message.split(';') module, num, keyword, paste_date = message.split(';')
redis_progression_name_set = 'top_'+ module +'_set' redis_progression_name_set = 'top_'+ module +'_set_' + paste_date
# Add/Update in Redis # Add/Update in Redis
prev_score = server.hget(paste_date, module+'-'+keyword) server.hincrby(paste_date, module+'-'+keyword, int(num))
if prev_score is not None:
ok = server.hset(paste_date, module+'-'+keyword, int(prev_score) + int(num))
else:
ok = server.hset(paste_date, module+'-'+keyword, int(num))
# Compute Most Posted # Compute Most Posted
date = get_date_range(0)[0] date = get_date_range(0)[0]
@ -47,103 +44,76 @@ def compute_most_posted(server, message):
curr_value = server.hget(date, module+'-'+keyword) curr_value = server.hget(date, module+'-'+keyword)
keyword_total_sum += int(curr_value) if curr_value is not None else 0 keyword_total_sum += int(curr_value) if curr_value is not None else 0
if keyword in server.smembers(redis_progression_name_set): # if it is already in the set if server.zcard(redis_progression_name_set) < max_set_cardinality:
return server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
if (server.scard(redis_progression_name_set) < max_set_cardinality): else: # not in set
server.sadd(redis_progression_name_set, keyword) member_set = server.zrangebyscore(redis_progression_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
else: #not in the set if int(member_set[0][1]) < keyword_total_sum:
#Check value for all members #remove min from set and add the new one
member_set = [] print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
for keyw in server.smembers(redis_progression_name_set): server.zrem(redis_progression_name_set, member_set[0][0])
keyw_value = server.hget(paste_date, module+'-'+keyw) server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
if keyw_value is not None: print redis_progression_name_set
member_set.append((keyw, int(keyw_value)))
else: #No data for this set for today
member_set.append((keyw, int(0)))
member_set.sort(key=lambda tup: tup[1])
if len(member_set) > 0:
if member_set[0][1] < keyword_total_sum:
#remove min from set and add the new one
print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server.srem(redis_progression_name_set, member_set[0][0])
server.sadd(redis_progression_name_set, keyword)
def compute_provider_info(server, path): def compute_provider_info(server_trend, server_pasteName, path):
redis_all_provider = 'all_provider_set'
redis_avg_size_name_set = 'top_size_set'
redis_providers_name_set = 'providers_set'
paste = Paste.Paste(path) paste = Paste.Paste(path)
paste_baseName = paste.p_name.split('.')[0]
paste_size = paste._get_p_size() paste_size = paste._get_p_size()
paste_provider = paste.p_source paste_provider = paste.p_source
paste_date = paste._get_p_date() paste_date = str(paste._get_p_date())
new_avg = paste_size redis_sum_size_set = 'top_size_set_' + paste_date
redis_avg_size_name_set = 'top_avg_size_set_' + paste_date
redis_providers_name_set = 'providers_set_' + paste_date
# Add/Update in Redis # Add/Update in Redis
prev_num_paste = server.hget(paste_provider+'_num', paste_date) server_pasteName.sadd(paste_baseName, path)
if prev_num_paste is not None: server_trend.sadd(redis_all_provider, paste_provider)
ok = server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1)
prev_sum_size = server.hget(paste_provider+'_size', paste_date)
if prev_sum_size is not None: num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
ok = server.hset(paste_provider+'_size', paste_date, float(prev_sum_size)+paste_size) sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size))
new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1) new_avg = float(sum_size) / float(num_paste)
else: server_trend.hset(paste_provider +'_avg', paste_date, new_avg)
ok = server.hset(paste_provider+'_size', paste_date, paste_size)
else:
ok = server.hset(paste_provider+'_num', paste_date, 1)
prev_num_paste = 0
# #
# Compute Most Posted # Compute Most Posted
# #
# Size # Size
if paste_provider not in server.smembers(redis_avg_size_name_set): # if it is already in the set if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil":
if (server.scard(redis_avg_size_name_set) < max_set_cardinality): server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider)
server.sadd(redis_avg_size_name_set, paste_provider) server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
else: #set full capacity
member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
if float(member_set[0][1]) < new_avg:
#remove min from set and add the new one
print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server_trend.zrem(redis_sum_size_set, member_set[0][0])
server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
else: #set full capacity
#Check value for all members
member_set = []
for provider in server.smembers(redis_avg_size_name_set):
curr_avg = 0.0
curr_size = server.hget(provider+'_size', paste_date)
curr_num = server.hget(provider+'_num', paste_date)
if (curr_size is not None) and (curr_num is not None):
curr_avg = float(curr_size) / float(curr_num)
member_set.append((provider, curr_avg))
member_set.sort(key=lambda tup: tup[1])
if member_set[0][1] < new_avg:
#remove min from set and add the new one
print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server.srem(redis_avg_size_name_set, member_set[0][0])
server.sadd(redis_avg_size_name_set, paste_provider)
# Num # Num
if paste_provider not in server.smembers(redis_providers_name_set): # if it is already in the set # if set not full or provider already present
if (server.scard(redis_providers_name_set) < max_set_cardinality): if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil":
server.sadd(redis_providers_name_set, paste_provider) server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
else: #set at full capacity
member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
if int(member_set[0][1]) < num_paste:
#remove min from set and add the new one
print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server_trend.zrem(member_set[0][0])
server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
else: #set full capacity
#Check value for all members
member_set = []
for provider in server.smembers(redis_providers_name_set):
curr_num = 0
curr_num = server.hget(provider+'_num', paste_date)
if curr_num is not None:
member_set.append((provider, int(curr_num)))
member_set.sort(key=lambda tup: tup[1])
if len(member_set) > 0:
if member_set[0][1] < int(prev_num_paste)+1:
#remove min from set and add the new one
print 'Num - adding ' +paste_provider+ '(' +str(int(prev_num_paste)+1)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server.srem(redis_providers_name_set, member_set[0][0])
server.sadd(redis_providers_name_set, paste_provider)
if __name__ == '__main__': if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@ -167,6 +137,11 @@ if __name__ == '__main__':
port=p.config.get("Redis_Level_DB_Trending", "port"), port=p.config.get("Redis_Level_DB_Trending", "port"),
db=p.config.get("Redis_Level_DB_Trending", "db")) db=p.config.get("Redis_Level_DB_Trending", "db"))
r_serv_pasteName = redis.StrictRedis(
host=p.config.get("Redis_Paste_Name", "host"),
port=p.config.get("Redis_Paste_Name", "port"),
db=p.config.get("Redis_Paste_Name", "db"))
# Endless loop getting messages from the input queue # Endless loop getting messages from the input queue
while True: while True:
# Get one message from the input queue # Get one message from the input queue
@ -183,4 +158,4 @@ if __name__ == '__main__':
if len(message.split(';')) > 1: if len(message.split(';')) > 1:
compute_most_posted(r_serv_trend, message) compute_most_posted(r_serv_trend, message)
else: else:
compute_provider_info(r_serv_trend, message) compute_provider_info(r_serv_trend, r_serv_pasteName, message)

143
bin/SentimentAnalyser.py Executable file
View file

@ -0,0 +1,143 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
Sentiment analyser module.
It takes its inputs from 'global'.
The content analysed comes from the pastes with length of the line
above a defined threshold removed (get_p_content_with_removed_lines).
This is done because NLTK sentences tokemnizer (sent_tokenize) seems to crash
for long lines (function _slices_from_text line#1276).
nltk.sentiment.vader module credit:
Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text. Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
"""
import time
import datetime
import calendar
import redis
import json
from pubsublogger import publisher
from Helper import Process
from packages import Paste
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk import tokenize
# Config Variables
accepted_Mime_type = ['text/plain']
size_threshold = 250
line_max_length_threshold = 1000
def Analyse(message, server):
path = message
paste = Paste.Paste(path)
# get content with removed line + number of them
num_line_removed, p_content = paste.get_p_content_with_removed_lines(line_max_length_threshold)
provider = paste.p_source
p_date = str(paste._get_p_date())
p_MimeType = paste._get_p_encoding()
# Perform further analysis
if p_MimeType == "text/plain":
if isJSON(p_content):
p_MimeType = "JSON"
if p_MimeType in accepted_Mime_type:
the_date = datetime.date(int(p_date[0:4]), int(p_date[4:6]), int(p_date[6:8]))
the_time = datetime.datetime.now()
the_time = datetime.time(getattr(the_time, 'hour'), 0, 0)
combined_datetime = datetime.datetime.combine(the_date, the_time)
timestamp = calendar.timegm(combined_datetime.timetuple())
sentences = tokenize.sent_tokenize(p_content.decode('utf-8', 'ignore'))
if len(sentences) > 0:
avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0}
neg_line = 0
pos_line = 0
sid = SentimentIntensityAnalyzer()
for sentence in sentences:
ss = sid.polarity_scores(sentence)
for k in sorted(ss):
if k == 'compound':
if ss['neg'] > ss['pos']:
avg_score['compoundNeg'] += ss[k]
neg_line += 1
else:
avg_score['compoundPos'] += ss[k]
pos_line += 1
else:
avg_score[k] += ss[k]
for k in avg_score:
if k == 'compoundPos':
avg_score[k] = avg_score[k] / (pos_line if pos_line > 0 else 1)
elif k == 'compoundNeg':
avg_score[k] = avg_score[k] / (neg_line if neg_line > 0 else 1)
else:
avg_score[k] = avg_score[k] / len(sentences)
# In redis-levelDB: {} = set, () = K-V
# {Provider_set -> provider_i}
# {Provider_TimestampInHour_i -> UniqID_i}_j
# (UniqID_i -> PasteValue_i)
server.sadd('Provider_set', provider)
provider_timestamp = provider + '_' + str(timestamp)
server.incr('UniqID')
UniqID = server.get('UniqID')
print provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines'
server.sadd(provider_timestamp, UniqID)
server.set(UniqID, avg_score)
else:
print 'Dropped:', p_MimeType
def isJSON(content):
try:
json.loads(content)
return True
except Exception,e:
return False
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'SentimentAnalysis'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("<description of the module>")
# REDIS_LEVEL_DB #
server = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_Sentiment", "host"),
port=p.config.get("Redis_Level_DB_Sentiment", "port"),
db=p.config.get("Redis_Level_DB_Sentiment", "db"))
while True:
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
Analyse(message, server)

View file

@ -23,16 +23,11 @@ num_day_to_look = 5 # the detection of the progression start num_day_to_lo
def analyse(server, field_name, date, url_parsed): def analyse(server, field_name, date, url_parsed):
field = url_parsed[field_name] field = url_parsed[field_name]
if field is not None: if field is not None:
prev_score = server.hget(field, date) server.hincrby(field, date, 1)
if prev_score is not None: if field_name == "domain": #save domain in a set for the monthly plot
server.hset(field, date, int(prev_score) + 1) domain_set_name = "domain_set_" + date[0:6]
server.sadd(domain_set_name, field)
else: print "added in " + domain_set_name +": "+ field
server.hset(field, date, 1)
if field_name == "domain": #save domain in a set for the monthly plot
domain_set_name = "domain_set_" + date[0:6]
server.sadd(domain_set_name, field)
print "added in " + domain_set_name +": "+ field
def get_date_range(num_day): def get_date_range(num_day):
curr_date = datetime.date.today() curr_date = datetime.date.today()
@ -81,9 +76,10 @@ def compute_progression(server, field_name, num_day, url_parsed):
#Check value for all members #Check value for all members
member_set = [] member_set = []
for keyw in server.smembers(redis_progression_name_set): for keyw in server.smembers(redis_progression_name_set):
member_set += (keyw, int(server.hget(redis_progression_name, keyw))) member_set.append((keyw, int(server.hget(redis_progression_name, keyw))))
print member_set
member_set.sort(key=lambda tup: tup[1]) member_set.sort(key=lambda tup: tup[1])
if member_set[0] < keyword_increase: if member_set[0][1] < keyword_increase:
#remove min from set and add the new one #remove min from set and add the new one
server.srem(redis_progression_name_set, member_set[0]) server.srem(redis_progression_name_set, member_set[0])
server.sadd(redis_progression_name_set, keyword) server.sadd(redis_progression_name_set, keyword)
@ -106,11 +102,6 @@ if __name__ == '__main__':
publisher.info("Makes statistics about valid URL") publisher.info("Makes statistics about valid URL")
# REDIS # # REDIS #
r_serv1 = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
db=p.config.get("Redis_Level_DB", "db"))
r_serv_trend = redis.StrictRedis( r_serv_trend = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_Trending", "host"), host=p.config.get("Redis_Level_DB_Trending", "host"),
port=p.config.get("Redis_Level_DB_Trending", "port"), port=p.config.get("Redis_Level_DB_Trending", "port"),

View file

@ -24,6 +24,7 @@ if __name__ == "__main__":
publisher.channel = "Script" publisher.channel = "Script"
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys'] config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
config_section = ['Curve']
for queue in config_section: for queue in config_section:
print 'dropping: ' + queue print 'dropping: ' + queue

View file

@ -7,13 +7,17 @@ lvdbhost='127.0.0.1'
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013' db1_y='2013'
db2_y='2014' db2_y='2014'
db2_y='2015' db3_y='2016'
db2_y='2016' db4_y='3016'
nb_db=13 nb_db=13
screen -dmS "LevelDB" screen -dmS "LevelDB"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
#Add lines here with appropriates options. sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c '../redis-leveldb/redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x' screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
# For Curve
sleep 0.1
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'

View file

@ -14,3 +14,6 @@ screen -S "Redis" -X screen -t "6380" bash -c '../redis/src/redis-server '$conf_
sleep 0.1 sleep 0.1
screen -S "Redis" -X screen -t "6381" bash -c '../redis/src/redis-server '$conf_dir'6381.conf ; read x' screen -S "Redis" -X screen -t "6381" bash -c '../redis/src/redis-server '$conf_dir'6381.conf ; read x'
# For Words and curves
sleep 0.1
screen -S "Redis" -X screen -t "6382" bash -c '../redis/src/redis-server '$conf_dir'6382.conf ; read x'

View file

@ -10,7 +10,7 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x' screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate_ssdeep_v2.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
sleep 0.1 sleep 0.1
@ -34,6 +34,24 @@ screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x'

View file

@ -91,6 +91,7 @@ class Paste(object):
self.p_langage = None self.p_langage = None
self.p_nb_lines = None self.p_nb_lines = None
self.p_max_length_line = None self.p_max_length_line = None
self.array_line_above_threshold = None
self.p_duplicate = None self.p_duplicate = None
def get_p_content(self): def get_p_content(self):
@ -118,6 +119,21 @@ class Paste(object):
def get_p_content_as_file(self): def get_p_content_as_file(self):
return cStringIO.StringIO(self.get_p_content()) return cStringIO.StringIO(self.get_p_content())
def get_p_content_with_removed_lines(self, threshold):
num_line_removed = 0
line_length_threshold = threshold
string_content = ""
f = self.get_p_content_as_file()
line_id = 0
for line_id, line in enumerate(f):
length = len(line)
if length < line_length_threshold:
string_content += line
else:
num_line_removed+=1
return (num_line_removed, string_content)
def get_lines_info(self): def get_lines_info(self):
""" """
Returning and setting the number of lines and the maximum lenght of the Returning and setting the number of lines and the maximum lenght of the
@ -136,9 +152,11 @@ class Paste(object):
length = len(line) length = len(line)
if length >= max_length_line: if length >= max_length_line:
max_length_line = length max_length_line = length
f.close() f.close()
self.p_nb_lines = line_id self.p_nb_lines = line_id
self.p_max_length_line = max_length_line self.p_max_length_line = max_length_line
return (self.p_nb_lines, self.p_max_length_line) return (self.p_nb_lines, self.p_max_length_line)
def _get_p_encoding(self): def _get_p_encoding(self):

View file

@ -1,6 +1,5 @@
[Directories] [Directories]
bloomfilters = Blooms bloomfilters = Blooms
#Duplicate_ssdeep
dicofilters = Dicos dicofilters = Dicos
pastes = PASTES pastes = PASTES
@ -59,11 +58,26 @@ host = localhost
port = 6379 port = 6379
db = 1 db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
##### LevelDB ##### ##### LevelDB #####
[Redis_Level_DB_Curve] [Redis_Level_DB_Curve]
host = localhost host = localhost
port = 3016 port = 6382
db = 0 db = 1
[Redis_Level_DB_Sentiment]
host = localhost
port = 6382
db = 4
[Redis_Level_DB_TermFreq]
host = localhost
port = 6382
db = 2
[Redis_Level_DB] [Redis_Level_DB]
host = localhost host = localhost
@ -72,8 +86,8 @@ db = 0
[Redis_Level_DB_Trending] [Redis_Level_DB_Trending]
host = localhost host = localhost
port = 2016 port = 6382
db = 0 db = 3
[Redis_Level_DB_Hashs] [Redis_Level_DB_Hashs]
host = localhost host = localhost

View file

@ -24,6 +24,10 @@ publish = Redis_Words
[Curve] [Curve]
subscribe = Redis_Words subscribe = Redis_Words
publish = Redis_CurveManageTopSets
[CurveManageTopSets]
subscribe = Redis_CurveManageTopSets
[Categ] [Categ]
subscribe = Redis_Global subscribe = Redis_Global
@ -66,6 +70,9 @@ subscribe = Redis_BrowseWarningPaste
#subscribe = Redis_Cve #subscribe = Redis_Cve
#publish = Redis_BrowseWarningPaste #publish = Redis_BrowseWarningPaste
[SentimentAnalysis]
subscribe = Redis_Global
[Release] [Release]
subscribe = Redis_Global subscribe = Redis_Global

View file

@ -42,7 +42,7 @@ popd
# tlsh # tlsh
test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
pushd tlsh/ pushd tlsh/
./make ./make.sh
popd popd
# REDIS LEVEL DB # # REDIS LEVEL DB #

View file

@ -5,6 +5,8 @@ import redis
import ConfigParser import ConfigParser
import json import json
import datetime import datetime
import time
import calendar
from flask import Flask, render_template, jsonify, request from flask import Flask, render_template, jsonify, request
import flask import flask
import os import os
@ -49,6 +51,22 @@ r_serv_db = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB", "port"), port=cfg.getint("Redis_Level_DB", "port"),
db=cfg.getint("Redis_Level_DB", "db")) db=cfg.getint("Redis_Level_DB", "db"))
r_serv_sentiment = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_Sentiment", "host"),
port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
r_serv_term = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_TermFreq", "host"),
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
r_serv_pasteName = redis.StrictRedis(
host=cfg.get("Redis_Paste_Name", "host"),
port=cfg.getint("Redis_Paste_Name", "port"),
db=cfg.getint("Redis_Paste_Name", "db"))
app = Flask(__name__, static_url_path='/static/') app = Flask(__name__, static_url_path='/static/')
@ -160,6 +178,7 @@ def showpaste(content_range):
if content_range != 0: if content_range != 0:
p_content = p_content[0:content_range] p_content = p_content[0:content_range]
return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list) return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
def getPastebyType(server, module_name): def getPastebyType(server, module_name):
@ -181,23 +200,32 @@ def get_date_range(num_day):
# Iterate over elements in the module provided and return the today data or the last data # Iterate over elements in the module provided and return the today data or the last data
# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]] # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
def get_top_relevant_data(server, module_name): def get_top_relevant_data(server, module_name):
redis_progression_name_set = 'top_'+ module_name +'_set'
days = 0 days = 0
for date in get_date_range(15): for date in get_date_range(15):
member_set = [] redis_progression_name_set = 'top_'+ module_name +'_set_' + date
for keyw in server.smembers(redis_progression_name_set): member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
redis_progression_name = module_name+'-'+keyw if len(member_set) == 0: #No data for this date
keyw_value = server.hget(date ,redis_progression_name)
keyw_value = keyw_value if keyw_value is not None else 0
member_set.append((keyw, int(keyw_value)))
member_set.sort(key=lambda tup: tup[1], reverse=True)
if member_set[0][1] == 0: #No data for this date
days += 1 days += 1
continue
else: else:
member_set.insert(0, ("passed_days", days)) member_set.insert(0, ("passed_days", days))
return member_set return member_set
def Term_getValueOverRange(word, startDate, num_day):
passed_days = 0
oneDay = 60*60*24
to_return = []
curr_to_return = 0
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
value = r_serv_term.hget(timestamp, word)
curr_to_return += int(value) if value is not None else 0
for i in num_day:
if passed_days == i-1:
to_return.append(curr_to_return)
passed_days += 1
return to_return
# ========= CACHE CONTROL ======== # ========= CACHE CONTROL ========
@app.after_request @app.after_request
def add_header(response): def add_header(response):
@ -230,7 +258,7 @@ def progressionCharts():
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value = r_serv_charts.hget(attribute_name, date) curr_value = r_serv_charts.hget(attribute_name, date)
@ -263,7 +291,7 @@ def modulesCharts():
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name) curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
@ -288,15 +316,14 @@ def providersChart():
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date) curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date)
curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date) curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
if module_name == "size": if module_name == "size":
curr_value_num = curr_value_num if curr_value_num is not None else 0 curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
curr_value_num = curr_value_num if int(curr_value_num) != 0 else 10000000000
curr_value = float(curr_value_size if curr_value_size is not None else 0.0) / float(curr_value_num)
else: else:
curr_value = float(curr_value_num if curr_value_num is not None else 0.0) curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
@ -305,30 +332,12 @@ def providersChart():
return jsonify(bar_values) return jsonify(bar_values)
else: else:
redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set' #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
# Iterate over element in top_x_set and retreive their value member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
member_set = [] # Member set is a list of (value, score) pairs
for keyw in r_serv_charts.smembers(redis_provider_name_set):
redis_provider_name_size = keyw+'_'+'size'
redis_provider_name_num = keyw+'_'+'num'
keyw_value_size = r_serv_charts.hget(redis_provider_name_size, get_date_range(0)[0])
keyw_value_size = keyw_value_size if keyw_value_size is not None else 0.0
keyw_value_num = r_serv_charts.hget(redis_provider_name_num, get_date_range(0)[0])
if keyw_value_num is not None:
keyw_value_num = int(keyw_value_num)
else:
if module_name == "size":
keyw_value_num = 10000000000
else:
keyw_value_num = 0
if module_name == "size":
member_set.append((keyw, float(keyw_value_size)/float(keyw_value_num)))
else:
member_set.append((keyw, float(keyw_value_num)))
member_set.sort(key=lambda tup: tup[1], reverse=True)
if len(member_set) == 0: if len(member_set) == 0:
member_set.append(("No relevant data", float(100))) member_set.append(("No relevant data", float(100)))
return jsonify(member_set) return jsonify(member_set)
@ -344,7 +353,22 @@ def search():
c = [] #preview of the paste content c = [] #preview of the paste content
paste_date = [] paste_date = []
paste_size = [] paste_size = []
# Search
# Search filename
print r_serv_pasteName.smembers(q[0])
for path in r_serv_pasteName.smembers(q[0]):
print path
r.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
c.append(content[0:content_range])
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_size.append(paste._get_p_size())
# Search full line
from whoosh import index from whoosh import index
from whoosh.fields import Schema, TEXT, ID from whoosh.fields import Schema, TEXT, ID
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
@ -429,6 +453,299 @@ def importantPasteByModule():
def moduletrending(): def moduletrending():
return render_template("Moduletrending.html") return render_template("Moduletrending.html")
@app.route("/sentiment_analysis_trending/")
def sentiment_analysis_trending():
return render_template("sentiment_analysis_trending.html")
@app.route("/sentiment_analysis_getplotdata/")
def sentiment_analysis_getplotdata():
# Get the top providers based on number of pastes
oneHour = 60*60
sevenDays = oneHour*24*7
dateStart = datetime.datetime.now()
dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
dateStart_timestamp = calendar.timegm(dateStart.timetuple())
to_return = {}
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
# if empty, get yesterday top providers
print 'providers_set_'+ get_date_range(1)[1]
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
# if still empty, takes from all providers
if range_providers == []:
print 'today provider empty'
range_providers = r_serv_charts.smembers('all_provider_set')
for cur_provider in range_providers:
print cur_provider
cur_provider_name = cur_provider + '_'
list_date = {}
for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = r_serv_sentiment.get(cur_id)
list_value.append(cur_value)
list_date[cur_timestamp] = list_value
to_return[cur_provider] = list_date
return jsonify(to_return)
@app.route("/sentiment_analysis_plot_tool/")
def sentiment_analysis_plot_tool():
return render_template("sentiment_analysis_plot_tool.html")
@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
def sentiment_analysis_plot_tool_getdata():
getProviders = request.args.get('getProviders')
if getProviders == 'True':
providers = []
for cur_provider in r_serv_charts.smembers('all_provider_set'):
providers.append(cur_provider)
return jsonify(providers)
else:
query = request.args.get('query')
query = query.split(',')
Qdate = request.args.get('Qdate')
date1 = (Qdate.split('-')[0]).split('.')
date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0]))
date2 = (Qdate.split('-')[1]).split('.')
date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0]))
timestamp1 = calendar.timegm(date1.timetuple())
timestamp2 = calendar.timegm(date2.timetuple())
oneHour = 60*60
oneDay = oneHour*24
to_return = {}
for cur_provider in query:
list_date = {}
cur_provider_name = cur_provider + '_'
for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = r_serv_sentiment.get(cur_id)
list_value.append(cur_value)
list_date[cur_timestamp] = list_value
to_return[cur_provider] = list_date
return jsonify(to_return)
@app.route("/terms_management/")
def terms_management():
TrackedTermsSet_Name = "TrackedSetTermSet"
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsDate_Name = "TrackedTermDate"
BlackListTermsDate_Name = "BlackListTermDate"
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
track_list = []
track_list_values = []
track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
track_list.append(tracked_term)
value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])
term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
set_paste_name = "tracked_" + tracked_term
track_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(term_date)
track_list_values.append(value_range)
black_list = []
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
black_list.append([blacked_term, term_date])
return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste)
@app.route("/terms_management_query_paste/")
def terms_management_query_paste():
term = request.args.get('term')
TrackedTermsSet_Name = "TrackedSetTermSet"
paste_info = []
set_paste_name = "tracked_" + term
track_list_path = r_serv_term.smembers(set_paste_name)
for path in track_list_path:
paste = Paste.Paste(path)
p_date = str(paste._get_p_date())
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
p_source = paste.p_source
p_encoding = paste._get_p_encoding()
p_size = paste.p_size
p_mime = paste.p_mime
p_lineinfo = paste.get_lines_info()
p_content = paste.get_p_content().decode('utf-8', 'ignore')
if p_content != 0:
p_content = p_content[0:400]
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
return jsonify(paste_info)
@app.route("/terms_management_query/")
def terms_management_query():
TrackedTermsDate_Name = "TrackedTermDate"
BlackListTermsDate_Name = "BlackListTermDate"
term = request.args.get('term')
section = request.args.get('section')
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
if section == "followTerm":
term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
elif section == "blacklistTerm":
term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(str(term_date))
return jsonify(value_range)
@app.route("/terms_management_action/", methods=['GET'])
def terms_management_action():
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedTermsDate_Name = "TrackedTermDate"
BlackListTermsDate_Name = "BlackListTermDate"
BlackListTermsSet_Name = "BlackListSetTermSet"
today = datetime.datetime.now()
today = today.replace(microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
section = request.args.get('section')
action = request.args.get('action')
term = request.args.get('term')
if action is None or term is None:
return "None"
else:
if section == "followTerm":
if action == "add":
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp)
else:
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
elif section == "blacklistTerm":
if action == "add":
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
else:
r_serv_term.srem(BlackListTermsSet_Name, term.lower())
else:
return "None"
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = term
return jsonify(to_return)
@app.route("/terms_plot_tool/")
def terms_plot_tool():
term = request.args.get('term')
if term is not None:
return render_template("terms_plot_tool.html", term=term)
else:
return render_template("terms_plot_tool.html", term="")
@app.route("/terms_plot_tool_data/")
def terms_plot_tool_data():
oneDay = 60*60*24
range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
range_start = calendar.timegm(range_start.timetuple())
range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
range_end = calendar.timegm(range_end.timetuple())
term = request.args.get('term')
if term is None:
return "None"
else:
value_range = []
for timestamp in range(range_start, range_end+oneDay, oneDay):
print timestamp, term
value = r_serv_term.hget(timestamp, term)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
return jsonify(value_range)
@app.route("/terms_plot_top/")
def terms_plot_top():
return render_template("terms_plot_top.html")
@app.route("/terms_plot_top_data/")
def terms_plot_top_data():
oneDay = 60*60*24
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
set_day = "TopTermFreq_set_day_" + str(today_timestamp)
set_week = "TopTermFreq_set_week";
set_month = "TopTermFreq_set_month";
the_set = request.args.get('set')
num_day = int(request.args.get('num_day'))
if the_set is None:
return "None"
else:
to_return = []
if the_set == "TopTermFreq_set_day":
the_set += "_" + str(today_timestamp)
for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
position = {}
position['day'] = r_serv_term.zrevrank(set_day, term)
position['day'] = position['day']+1 if position['day'] is not None else "<20"
position['week'] = r_serv_term.zrevrank(set_week, term)
position['week'] = position['week']+1 if position['week'] is not None else "<20"
position['month'] = r_serv_term.zrevrank(set_month, term)
position['month'] = position['month']+1 if position['month'] is not None else "<20"
value_range = []
for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
value = r_serv_term.hget(timestamp, term)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
to_return.append([term, value_range, tot_value, position])
return jsonify(to_return)
@app.route("/showsavedpaste/") #completely shows the paste in a new tab @app.route("/showsavedpaste/") #completely shows the paste in a new tab
def showsavedpaste(): def showsavedpaste():

View file

@ -0,0 +1,68 @@
.switch {
position: relative;
display: inline-block;
vertical-align: top;
width: 56px;
height: 20px;
padding: 3px;
margin-left: 5px;
border-radius: 18px;
box-shadow: inset 0 -1px #fff,inset 0 1px 1px rgba(0,0,0,0.05);
cursor: pointer;
-webkit-box-sizing: content-box;
-moz-box-sizing: content-box;
box-sizing: content-box;
}
.switch-input {
position: absolute;
top: 0;
left: 0;
opacity: 0;
}
.switch-input:checked ~ .switch-label {
background: #67c2ef;
box-shadow: inset 0 1px 2px rgba(0,0,0,0.15),inset 0 0 3px rgba(0,0,0,0.2);
}
.switch-label {
position: relative;
display: block;
height: inherit;
font-size: 10px;
text-transform: uppercase;
background: #f9f9f9;
border-radius: inherit;
box-shadow: inset 0 1px 2px rgba(0,0,0,0.12),inset 0 0 2px rgba(0,0,0,0.15);
-webkit-transition: .15s ease-out;
-moz-transition: .15s ease-out;
-o-transition: .15s ease-out;
transition: .15s ease-out;
-webkit-transition-property: opacity background;
-moz-transition-property: opacity background;
-o-transition-property: opacity background;
transition-property: opacity background;
}
.switch-input:checked ~ .switch-handle {
left: 40px;
box-shadow: -1px 1px 5px rgba(0,0,0,0.2);
}
.switch-handle {
position: absolute;
top: 4px;
left: 4px;
width: 18px;
height: 18px;
background: white;
border-radius: 10px;
box-shadow: 1px 1px 5px rgba(0,0,0,0.2);
background-image: -webkit-linear-gradient(top,#fff 40%,#f0f0f0);
background-image: -moz-linear-gradient(top,#fff 40%,#f0f0f0);
background-image: -o-linear-gradient(top,#fff 40%,#f0f0f0);
background-image: linear-gradient(to bottom,#fff 40%,#f0f0f0);
-webkit-transition: left .15s ease-out;
-moz-transition: left .15s ease-out;
-o-transition: left .15s ease-out;
transition: left .15s ease-out;
}

View file

@ -0,0 +1,440 @@
/**
* FlexGauge
* Version: 1.0
* Author: Jeff Millies
* Author URI:
*
* Slight modification for better display in Sentiment webpages
*/
(function ($) {
var FlexGauge = function (o) {
if (typeof o === 'object') {
this._extendOptions(o, false);
this._build();
}
};
FlexGauge.prototype = {
/**
* {String} Element that you would like to append to. ie '#idname', '.classname', 'div#idname', etc..
*/
appendTo: 'body',
/**
* {String} Id of Canvas already created or Id of canvas that will be created automatically
*/
elementId: 'canvas',
/**
* {String} Class of canvas created
*/
elementClass: 'canvas',
/**
* {Int} Canvas Width & Height
*/
elementWidth: 200,
elementHeight: 200,
/**
* {Boolean|String} Generate Dial Value for the Gauge, true will use arcFillPercent or arcFillInt
* depending on provided values and specified dialUnits, string will use specified value
*/
dialValue: false,
/**
* {String} Class applied to div when dial is generated.
*/
dialClass: 'fg-dial',
/**
* {string: %|$| } Type of unit to use for the dial
*/
dialUnit: '%',
/**
* {string: before|after} Where the dial unit will be displayed
*/
dialUnitPosition: 'after',
/**
* {Boolean|String} Generate Label for the Gauge, true will use default "FlexGauge", string will use specified
*/
dialLabel: false,
/**
* {String} Class applied to div when label is generated.
*/
dialLabelClass: 'fg-dial-label',
/**
* {Int} Radius of the arc
*/
inc: 0.0,
incTot: 1.0,
/**
* {Doule} Increment value
*/
arcSize: 85,
/**
* {double} Starting and Ending location of the arc, End always needs to be larger
* arc(x, y, radius, startAngle, endAngle, anticlockwise)
*/
arcAngleStart: 0.85,
arcAngleEnd: 2.15,
/**
* {double} Percentage the arc fills
*/
arcFillPercent: .5,
/**
* {Int} Starting and Ending values that are used to
* find a difference for amount of units
* ie: 60 (arcFillEnd) - 10 (arcFillStart) = 50
*/
arcFillStart: null,
arcFillEnd: null,
/**
* {Int} Data used to find out what percentage of the
* arc to fill. arcFillInt can be populated by
* the difference of arcFillStart and arcFillEnd
*/
arcFillInt: null,
arcFillTotal: null,
/**
* {Int} Color lightness: 0 - 255, 0 having no white added, 255 having all white and no color
*/
arcBgColorLight: 80,
/**
* {Int} Color saturation: 0 - 100, 0 having no color, 100 is full color
*/
arcBgColorSat: 60,
/**
* {Int} Size of the line marking the percentage
*/
arcStrokeFg: 30,
/**
* {Int} Size of the container holding the line
*/
arcStrokeBg: 30,
/**
* {string: hex} Color of the line marking the percentage
*/
colorArcFg: '#5bc0de',
/**
* {string: hex} Color of the container holding the line, default is using the Fg color and lightening it
*/
colorArcBg: null,
/**
* {String} Instead of providing a color or hex for the color, you can provide a class from the style
* sheet and specify what you would like to grab for the color in styleSrc
*/
styleArcFg: null,
styleArcBg: null,
styleSrc: 'color',
/**
* {Boolean} If set to false, then the graph will not be animated
*/
animateEasing: true,
/**
* {Int} Speed for the animation, 1 is fastest, higher the number, slower the animation
*/
animateSpeed: 5,
/**
* {Int} Math used in animation speed
*/
animateNumerator: 12,
animateDivisor: 15,
/**
* {double} Placeholder for current percentage while animating
*/
_animatePerc: 0.00,
/**
* {Object} Placeholder for setInterval
*/
_animateLoop: null,
/**
* {Object} Placeholder for canvas
*/
_canvas: null,
/**
* {Object} Placeholder for canvas context
*/
_ctx: null,
update: function (o) {
if (typeof o === 'object') {
var difference;
// if using int, convert to percent to check difference
if (typeof o.arcFillInt !== 'undefined' && o.arcFillInt == this.arcFillInt &&
typeof o.arcFillTotal !== 'undefined' && o.arcFillTotal == this.arcFillTotal) {
o.arcFillPercent = this.arcFillPercent;
} else if (typeof o.arcFillInt !== 'undefined' && typeof o.arcFillTotal !== 'undefined' &&
(o.arcFillInt != this.arcFillInt || o.arcFillTotal == this.arcFillTotal)) {
o.arcFillPercent = (o.arcFillInt / o.arcFillTotal);
} else if (typeof o.arcFillInt !== 'undefined' && typeof o.arcFillTotal === 'undefined' &&
(o.arcFillInt != this.arcFillInt)) {
o.arcFillPercent = (o.arcFillInt / this.arcFillTotal);
}
if (typeof o.arcFillPercent !== 'undefined') {
difference = Math.abs((this.arcFillPercent - o.arcFillPercent));
} else {
difference = this.arcFillPercent;
}
this._extendOptions(o, true);
clearInterval(this._animateLoop);
if (difference > 0) {
var that = this;
this._animateLoop = setInterval(function () {
return that._animate();
}, (this.animateSpeed * this.animateNumerator) / (difference * this.animateDivisor));
}
}
},
_extendOptions: function (o, update) {
var color = false;
if (update)
color = this.colorArcFg;
$.extend(this, o, true);
if (typeof o.arcFillStart !== 'undefined' && typeof o.arcFillEnd !== 'undefined' && typeof o.arcFillTotal !== 'undefined') {
this.arcFillInt = (o.arcFillEnd - o.arcFillStart);
}
if (typeof o.arcFillPercent === 'undefined' && this.arcFillInt !== null && this.arcFillInt >= 0 && this.arcFillTotal !== null && this.arcFillTotal > 0) {
this.arcFillPercent = this.arcFillInt / this.arcFillTotal;
}
if (typeof o.elementId === 'undefined') {
this.elementId = 'fg-' + this.appendTo + '-canvas';
}
// supporting color if pass, changing to hex
if (typeof o.colorArcFg !== 'undefined') {
this.colorArcFg = colorToHex(o.colorArcFg);
}
if (typeof o.colorArcBg !== 'undefined') {
this.colorArcBg = colorToHex(o.colorArcBg);
}
// only use the styleArcFg if colorArcFg wasn't specified in the options
if (typeof o.styleArcFg !== 'undefined' && typeof o.colorArcFg === 'undefined') {
this.colorArcFg = getStyleRuleValue(this.styleSrc, this.styleArcFg);
}
if (typeof o.colorArcBg === 'undefined' && this.colorArcBg === null && this.colorArcFg !== null) {
this.colorArcBg = this.colorArcFg;
}
if (typeof this.colorArcBg !== null && (!update || colorToHex(this.colorArcFg) != colorToHex(color))) {
if (colorToHex(this.colorArcFg) != colorToHex(color))
this.colorArcBg = this.colorArcFg;
this.colorArcBg = shadeColor(this.colorArcBg, this.arcBgColorLight, this.arcBgColorSat);
}
if (typeof o.dialLabel === 'boolean' && o.dialLabel) {
this.dialLabel = 'FlexGauge';
}
},
_build: function () {
if (document.getElementById(this.elementId) === null) {
$(this.appendTo).append('<canvas id="' + this.elementId + '" width="' + this.elementWidth + '" height="' + this.elementHeight + '"></canvas>');
}
this._canvas = document.getElementById(this.elementId);
this._ctx = this._canvas.getContext("2d");
this.arcAngleStart = this.arcAngleStart * Math.PI;
this.arcAngleEnd = this.arcAngleEnd * Math.PI;
if (this.animateEasing === false) {
this._animatePerc = this.arcFillPercent;
}
var that = this;
this._animateLoop = setInterval(function () {
return that._animate();
}, (this.animateSpeed * this.animateNumerator) / (this.arcFillPercent * this.animateDivisor));
},
_animate: function () {
var animateInt = Math.round(this._animatePerc * 100);
var arcInt = Math.round(this.arcFillPercent * 100);
if (animateInt < arcInt)
animateInt++;
else
animateInt--;
this._animatePerc = (animateInt / 100);
if (animateInt === arcInt) {
this.arcFillPercent = this._animatePerc;
clearInterval(this._animateLoop);
this._draw();
}
this._draw();
},
_draw: function () {
//Clear the canvas everytime a chart is drawn
this._ctx.clearRect(0, 0, this.elementWidth, this.elementHeight);
//Background 360 degree arc
this._ctx.beginPath();
this._ctx.strokeStyle = this.colorArcBg;
this._ctx.lineWidth = this.arcStrokeBg;
this._ctx.arc(
this.elementWidth / 2,
this.elementHeight / 2 + 50,
this.arcSize,
0,
Math.PI,
true
);
this._ctx.stroke();
//var newEnd = ((this.arcAngleEnd - this.arcAngleStart) * this._animatePerc) + this.arcAngleStart;
var newStart;
var newEnd;
var incArc = this.inc*Math.PI/2;
if (this.inc >= 0.0){
newStart = -Math.PI/2;
newEnd = newStart + incArc;
} else {
newStart = -Math.PI/2 + incArc;
newEnd = -Math.PI/2;
}
var colorShadesTabRed = ['#ff0000','#ff4000','#ff8000','#ff9900','#ffbf00','#ffff00'];
var colorShadesTabGreen = ['#ffff00','#E0FF00','#D0FF00','#a0ff00','#00ff00','#00ff40',];
var colorValue = parseInt(Math.abs((this.inc / this.incTot) * 5));
var theColor;
if (this.inc >= 0.0)
theColor = colorShadesTabGreen[colorValue];
else
theColor = colorShadesTabRed[5-colorValue];
this.colorArcFg = theColor;
this._ctx.beginPath();
this._ctx.strokeStyle = this.colorArcFg;
this._ctx.lineWidth = this.arcStrokeFg;
this._ctx.arc(
this.elementWidth / 2,
this.elementHeight / 2 + 50,
this.arcSize,
newStart,
newEnd,
false
);
this._ctx.stroke();
this._renderLabel();
},
_renderLabel: function () {
if (this.dialValue) {
var dialVal;
var dial = $(this.appendTo).find('div.' + this.dialClass);
if (dial.length === 0) {
$(this.appendTo).append('<div class="' + this.dialClass + '"></div>');
}
dial = $(this.appendTo).find('div.' + this.dialClass);
if (typeof this.dialValue === 'boolean') {
switch (this.dialUnit) {
case '%':
dialVal = Math.round(this._animatePerc * 100);
break;
default:
dialVal = Math.round(this.arcFillInt * (this._animatePerc / this.arcFillPercent));
break;
}
dialVal = (isNaN(dialVal) ? 0 : dialVal);
switch (this.dialUnitPosition) {
case 'before':
dialVal = this.dialUnit + dialVal;
break;
case 'after':
dialVal = dialVal + this.dialUnit;
break;
}
} else {
dialVal = this.dialValue;
}
dial.html(dialVal)
}
if (this.dialLabel) {
var label = $(this.appendTo).find('div.' + this.dialLabelClass);
if (label.length === 0) {
$(this.appendTo).append('<div class="' + this.dialLabelClass + '"></div>');
}
label = $(this.appendTo).find('div.' + this.dialLabelClass);
label.html(this.dialLabel);
}
}
};
function shadeColor(col, amt, sat) {
if (col[0] == "#") {
col = col.slice(1);
}
var num = parseInt(col, 16);
var r = (num >> 16) + amt;
if (r > 255) r = 255;
else if (r < 0) r = 0;
var b = ((num >> 8) & 0x00FF) + amt;
if (b > 255) b = 255;
else if (b < 0) b = 0;
var g = (num & 0x0000FF) + amt;
if (g > 255) g = 255;
else if (g < 0) g = 0;
var gray = r * 0.3086 + g * 0.6094 + b * 0.0820;
sat = (sat / 100);
r = Math.round(r * sat + gray * (1 - sat));
g = Math.round(g * sat + gray * (1 - sat));
b = Math.round(b * sat + gray * (1 - sat));
return "#" + (g | (b << 8) | (r << 16)).toString(16);
}
function getStyleRuleValue(style, selector) {
$('body').append('<div id="getStyleRuleValue-' + selector + '"></div>');
var element = $('#getStyleRuleValue-' + selector);
element.addClass(selector);
var color = element.css(style);
var hex = colorToHex(color);
element.remove();
return hex;
}
function colorToHex(color) {
if (color[0] != 'r')
return color;
var rgb = color.match(/^rgb\((\d+),\s*(\d+),\s*(\d+)\)$/);
return "#" +
("0" + parseInt(rgb[1], 10).toString(16)).slice(-2) +
("0" + parseInt(rgb[2], 10).toString(16)).slice(-2) +
("0" + parseInt(rgb[3], 10).toString(16)).slice(-2);
}
if (typeof define === 'function') {
define('flex-gauge', ['jquery'], function ($) {
return FlexGauge;
});
} else {
window.FlexGauge = FlexGauge;
}
})(jQuery);

View file

@ -309,7 +309,13 @@ $(document).ready(function () {
var tmp_values2 = []; var tmp_values2 = [];
refresh(); refresh();
update_values(); update_values();
create_queue_table();
if($('#button-toggle-queues').prop('checked')){
create_queue_table();
}
else{
$("#queueing").html('');
}
for (i = 0; i < (glob_tabvar.row1).length; i++){ for (i = 0; i < (glob_tabvar.row1).length; i++){

View file

@ -0,0 +1,94 @@
var li_text = "<li><div class='checkbox'></div><label class='provider'><input value='"
var li_text_mid = "' type='checkbox'></input> "
var li_text_end = "</label></li>"
/* Get Providers List and display them by row */
$.getJSON('/sentiment_analysis_plot_tool_getdata/?getProviders=True', function(data){
for(i=0; i<data.length; i++){
var providerList = i%2 == 0 ? '#providerList1' : '#providerList2';
$(providerList).append(li_text + data[i] + li_text_mid + data[i] + li_text_end);
}
});
/* Create the slider and button*/
var today = Date.now();
var old_day = today - (31*24*60*60)*1000;
$( ".sliderRange" ).slider({
range: true,
min: old_day,
max: today,
values: [ today - (7*24*60*60)*1000, today ],
step: 24*60*60*1000,
slide: function( event, ui ) {
$( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
}
});
$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
" - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
$('#plot_btn').click(plotData);
/* Plot the requested data (if available) stored in slider and checkboxes */
function plotData(){
var graph_options = {
series: {
stack: $('#checkbox_stacked').prop('checked'),
lines: { show: false,
lineWidth: 2,
fill: true, fillColor: { colors: [ { opacity: 0.5 }, { opacity: 0.2 } ] }
},
bars: {show: true, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
xaxis: {
mode: "time",
timeformat: "%m/%d:%Hh",
minTickSize: [1, "hour"]
},
yaxis: {
autoscaleMargin: 0.1,
},
}
var query = $( "input:checked[value]" ).map(function () {return this.value;}).get().join(",");
var Qdate = new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +'-'+ new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString();
// retreive the data from the server
$.getJSON('/sentiment_analysis_plot_tool_getdata/?getProviders=False&query='+query+'&Qdate='+Qdate, function(data){
var to_plot = [];
for (provider in data){
var nltk_data = Object.keys(data[provider]).map(function (key) { return data[provider][key]; });
var nltk_key = Object.keys(data[provider]).map(function (key) { return key; });
var pos = 0.0;
var neg = 0.0;
var XY_data = [];
for (i=0; i<nltk_data.length; i++){
if (nltk_data[i].length == 0)
continue;
else {
for (j=0; j<nltk_data[i].length; j++){
var curr_data = jQuery.parseJSON(nltk_data[i][j].replace(/\'/g, '\"'));
pos += curr_data['pos'];
neg += curr_data['neg'];
}
pos = pos/nltk_data.length;
neg = neg/nltk_data.length;
}
XY_data.push([nltk_key[i]*1000, pos-neg]);
}
to_plot.push({ data: XY_data, label: provider});
}
var plot = $.plot($("#graph"), to_plot, graph_options);
});
}

View file

@ -0,0 +1,386 @@
function generate_offset_to_time(num){
var to_ret = {};
for(i=0; i<=num; i++) {
var t1 = new Date().getHours()-(23-i);
t1 = t1 < 0 ? 24+t1 : t1;
to_ret[i] = t1+'h';
}
return to_ret;
};
function generate_offset_to_date(day){
day = day-1;
var now = new Date();
var to_ret = {};
for(i=day; i>=0; i--){
for(j=0; j<24; j++){
var t1 =now.getDate()-i + ":";
var t2 =now.getHours()-(23-j);
t2 = t2 < 0 ? 24+t2 : t2;
t2 += "h";
to_ret[j+24*(day-i)] = t1+t2;
}
}
return to_ret;
};
var offset_to_time = generate_offset_to_time(23);
var offset_to_date = generate_offset_to_date(7);
var sparklineOptions = {
height: 80,//Height of the chart - Defaults to 'auto' (line height of the containing tag)
chartRangeMin: -1,
chartRangeMax: 1,
type: 'bar',
barSpacing: 0,
barWidth: 2,
barColor: '#00bf5f',
negBarColor: '#f22929',
zeroColor: '#ffff00',
tooltipFormat: '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>',
};
$.getJSON("/sentiment_analysis_getplotdata/",
function(data) {
var all_data = [];
var plot_data = [];
var graph_avg = [];
var array_provider = Object.keys(data);
var dates_providers = Object.keys(data[array_provider[0]]);
var dateStart = parseInt(dates_providers[0]);
var oneHour = 60*60;
var oneWeek = oneHour*24*7;
var all_graph_day_sum = 0.0;
var all_graph_hour_sum = 0.0;
var all_graph_hour_maxVal = 0.0;
var all_day_avg = 0.0;
var all_day_avg_maxVal = 0.0;
for (graphNum=0; graphNum<8; graphNum++) {
var max_value = 0.0;
var max_value_day = 0.0;
var graph_data = [];
var spark_data = [];
var curr_provider = array_provider[graphNum];
var curr_sum = 0.0;
var curr_sum_elem = 0.0;
var day_sum = 0.0;
var day_sum_elem = 0.0;
var hour_sum = 0.0;
for(curr_date=dateStart+oneHour; curr_date<=dateStart+oneWeek; curr_date+=oneHour){
var data_array = data[curr_provider][curr_date];
if (data_array.length == 0){
graph_data.push({'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0});
spark_data.push(0);
} else { //compute avg for a given date for a given graph
var compPosAvg = 0;
var compNegAvg = 0;
var pos = 0;
var neg = 0;
var neu = 0;
for(i=0; i<data_array.length; i++){
//console.log(data_array[i].replace(/\'/g, '\"'));
var curr_data = jQuery.parseJSON(data_array[i].replace(/\'/g, '\"'));
compPosAvg += curr_data['compoundPos'];
compNegAvg += curr_data['compoundNeg'];
pos += curr_data['pos'];
neg += curr_data['neg'];
neu += curr_data['neu'];
}
compPosAvg = compPosAvg/data_array.length;
compNegAvg = compNegAvg/data_array.length;
pos = pos/data_array.length;
neg = neg/data_array.length;
neu = neu/data_array.length;
graph_data.push({'neg': neg, 'neu': neu, 'pos': pos, 'compoundPos': compPosAvg, 'compoundNeg': compNegAvg});
spark_data.push(pos-neg);
curr_sum += (pos-neg);
curr_sum_elem++;
max_value = Math.abs(pos-neg) > max_value ? Math.abs(pos-neg) : max_value;
if(curr_date >= dateStart+oneWeek-23*oneHour){
max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day;
day_sum += (pos-neg);
day_sum_elem++;
}
if(curr_date > dateStart+oneWeek-2*oneHour && curr_date <=dateStart+oneWeek-oneHour){
hour_sum += (pos-neg);
}
}
}
all_graph_day_sum += day_sum;
all_graph_hour_sum += hour_sum;
all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal;
var curr_avg = curr_sum / (curr_sum_elem);
if(isNaN(curr_avg))
curr_avg = 0.0
//var curr_avg = curr_sum / (oneWeek/oneHour);
//var curr_avg = curr_sum / (spark_data.length);
graph_avg.push([curr_provider, curr_avg]);
plot_data.push(spark_data);
all_data.push(graph_data);
sparklineOptions.chartRangeMax = max_value;
sparklineOptions.chartRangeMin = -max_value;
sparklineOptions.tooltipValueLookups = { names: offset_to_date};
// print week
var num = graphNum + 1;
var placeholder = '.sparkLineStatsWeek' + num;
sparklineOptions.barWidth = 2;
$(placeholder).sparkline(plot_data[graphNum], sparklineOptions);
$(placeholder+'t').text(curr_provider);
var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5);
$(placeholder+'s').text(curr_avg_text);
sparklineOptions.barWidth = 18;
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
$(placeholder+'b').sparkline([curr_avg], sparklineOptions);
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
sparklineOptions.tooltipValueLookups = { names: offset_to_time};
sparklineOptions.chartRangeMax = max_value_day;
sparklineOptions.chartRangeMin = -max_value_day;
var avgName = ".pannelWeek" + num;
if (curr_avg > 0) {
$(avgName).addClass("panel-success")
} else if(curr_avg < 0) {
$(avgName).addClass("panel-danger")
} else if(isNaN(curr_avg)) {
$(avgName).addClass("panel-info")
} else {
$(avgName).addClass("panel-warning")
}
// print today
var data_length = plot_data[graphNum].length;
var data_today = plot_data[graphNum].slice(data_length-24, data_length);
placeholder = '.sparkLineStatsToday' + num;
sparklineOptions.barWidth = 14;
$(placeholder).sparkline(data_today, sparklineOptions);
$(placeholder+'t').text(curr_provider);
sparklineOptions.barWidth = 18;
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
//var day_avg = day_sum/24;
var day_avg = isNaN(day_sum/day_sum_elem) ? 0 : day_sum/day_sum_elem;
var day_avg_text = isNaN(day_sum/day_sum_elem) ? 'No data' : (day_avg).toFixed(5);
all_day_avg += day_avg;
all_day_avg_maxVal = Math.abs(day_avg) > all_day_avg_maxVal ? Math.abs(day_avg) : all_day_avg_maxVal;
$(placeholder+'b').sparkline([day_avg], sparklineOptions);
sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
$(placeholder+'s').text(day_avg_text);
avgName = ".pannelToday" + num;
if (day_avg > 0) {
$(avgName).addClass("panel-success")
} else if(day_avg < 0) {
$(avgName).addClass("panel-danger")
} else if(isNaN(day_sum/day_sum_elem)) {
$(avgName).addClass("panel-info")
} else {
$(avgName).addClass("panel-warning")
}
}//for loop
/* ---------------- Gauge ---------------- */
var gaugeOptions = {
animateEasing: true,
elementWidth: 200,
elementHeight: 125,
arcFillStart: 10,
arcFillEnd: 12,
arcFillTotal: 20,
incTot: 1.0,
arcBgColorLight: 200,
arcBgColorSat: 0,
arcStrokeFg: 20,
arcStrokeBg: 30,
colorArcFg: '#FF3300',
animateSpeed: 1,
};
// Clone object
var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions);
var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions);
gaugeOptions.appendTo = '#gauge_today_last_hour';
gaugeOptions.dialLabel = 'Last hour';
gaugeOptions.elementId = 'gauge1';
var piePercent = (all_graph_hour_sum / 8) / all_graph_hour_maxVal;
gaugeOptions.inc = piePercent;
var gauge_today_last_hour = new FlexGauge(gaugeOptions);
gaugeOptions2.appendTo = '#gauge_today_last_days';
gaugeOptions2.dialLabel = 'Today';
gaugeOptions2.elementId = 'gauge2';
//piePercent = (all_graph_day_sum / (8*24)) / max_value;
piePercent = (all_day_avg / 8) / all_day_avg_maxVal;
gaugeOptions2.inc = piePercent;
var gauge_today_last_days = new FlexGauge(gaugeOptions2);
gaugeOptions3.appendTo = '#gauge_week';
gaugeOptions3.dialLabel = 'Week';
gaugeOptions3.elementId = 'gauge3';
var graph_avg_sum = 0.0;
var temp_max_val = 0.0;
for (i=0; i<graph_avg.length; i++){
graph_avg_sum += graph_avg[i][1];
temp_max_val = Math.abs(graph_avg[i][1]) > temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val;
}
piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val;
gaugeOptions3.inc = piePercent;
var gauge_today_last_days = new FlexGauge(gaugeOptions3);
/* --------- Sort providers -------- */
graph_avg.sort(function(a, b){return b[1]-a[1]});
for (i=1; i<6; i++){
$('.worst'+i).text(graph_avg[7-(i-1)][0]);
$('.best'+i).text(graph_avg[i-1][0]);
}
/* ----------- CanvasJS ------------ */
var comp_sum_day_pos = 0.0;
var comp_sum_day_neg = 0.0;
var comp_sum_hour_pos = 0.0;
var comp_sum_hour_neg = 0.0;
for(graphNum=0; graphNum<8; graphNum++){
curr_graphData = all_data[graphNum];
var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length);
for (i=1; i< gauge_data.length; i++){
comp_sum_day_pos += gauge_data[i].compoundPos;
comp_sum_day_neg += gauge_data[i].compoundNeg;
if(i == 23){
comp_sum_hour_pos += gauge_data[i].compoundPos;
comp_sum_hour_neg += gauge_data[i].compoundNeg;
}
}
}
var options_canvasJS_1 = {
animationEnabled: true,
axisY: {
tickThickness: 0,
lineThickness: 0,
valueFormatString: " ",
gridThickness: 0
},
axisX: {
tickThickness: 0,
lineThickness: 0,
labelFontSize: 0.1,
},
data: [
{
toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
type: "bar",
color: "green",
dataPoints: [
{y: comp_sum_hour_pos/8}
]
},
{
toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
type: "bar",
color: "red",
dataPoints: [
{y: comp_sum_hour_neg/8}
]
}
]
};
var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1);
var options_canvasJS_2 = {
animationEnabled: true,
axisY: {
tickThickness: 0,
lineThickness: 0,
valueFormatString: " ",
gridThickness: 0
},
axisX: {
tickThickness: 0,
lineThickness: 0,
labelFontSize: 0.1,
},
data: [
{
toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
type: "bar",
color: "green",
dataPoints: [
{y: comp_sum_day_pos/8}
]
},
{
toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
type: "bar",
color: "red",
dataPoints: [
{y: comp_sum_day_neg/8}
]
}
]
};
var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2);
chart_canvas1.render();
chart_canvas2.render();
}
);

View file

@ -4,5 +4,20 @@
<li id='page-trendingchart'><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li> <li id='page-trendingchart'><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
<li id='page-modulestats'><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li> <li id='page-modulestats'><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
<li id='page-browse'><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li> <li id='page-browse'><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
</ul>
</li>
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>
</li>
</ul> </ul>
</div> </div>

View file

@ -28,7 +28,7 @@
<script> <script>
$(document).ready(function(){ $(document).ready(function(){
$('[data-toggle="tooltip"]').tooltip(); $('[data-toggle="tooltip"]').tooltip();
$('#myTable').dataTable(); $('#myTable').dataTable({ "order": [[ 2, "desc" ]] });
}); });
</script> </script>
@ -92,7 +92,8 @@ $(document).ready(function(){
} }
if (final_index != start_index){ // still have data to display if (final_index != start_index){ // still have data to display
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1)); // Append the new content // Append the new content using text() and not append (XSS)
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
start_index = final_index; start_index = final_index;
if (flag_stop) if (flag_stop)
nothing_to_display(); nothing_to_display();
@ -115,7 +116,6 @@ $(document).ready(function(){
$('#myTable').on( 'draw.dt', function () { $('#myTable').on( 'draw.dt', function () {
// On click, get html content from url and update the corresponding modal // On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) { $("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) {
console.log('hi');
event.preventDefault(); event.preventDefault();
var modal=$(this); var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num'); var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');

View file

@ -12,6 +12,7 @@
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" /> <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<!-- JS --> <!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script> <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
@ -57,6 +58,14 @@
</div> </div>
</div> </div>
<!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> --> <!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
<div class='pull_right'>
<label class="switch">
<input id="button-toggle-queues" class="switch-input" type="checkbox" checked>
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">Display queues</strong>
<div>
<div class="table-responsive", id="queueing" style="margin-top:10px;"></div> <div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a> <a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div> </div>
@ -100,7 +109,7 @@
<option value="100">100</option> <option value="100">100</option>
</select> </select>
</label> </label>
<input id="checkbox_log_info" type="checkbox" value="info" checked="true"> INFO <input id="checkbox_log_info" type="checkbox" value="info"> INFO
<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING <input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL <input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL

View file

@ -23,12 +23,11 @@
.tooltip-inner { .tooltip-inner {
text-align: left; text-align: left;
height: 200%; height: 200%;
width: 200%;
max-width: 500px; max-width: 500px;
max-height: 500px; max-height: 500px;
font-size: 13px; font-size: 13px;
} }
xmp { pre {
white-space:pre-wrap; white-space:pre-wrap;
word-wrap:break-word; word-wrap:break-word;
} }
@ -42,12 +41,7 @@
<div id="wrapper"> <div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0"> <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header"> {% include 'header.html' %}
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
<li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
</ul>
</div>
<!-- /.navbar-top-links --> <!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation"> <div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse"> <div class="sidebar-collapse">
@ -201,7 +195,8 @@
} }
if (final_index != start_index){ // still have data to display if (final_index != start_index){ // still have data to display
$("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content // Append the new content using text() and not append (XSS)
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text() + complete_paste.substring(start_index+1, final_index+1));
start_index = final_index; start_index = final_index;
if (flag_stop) if (flag_stop)
nothing_to_display(); nothing_to_display();

View file

@ -0,0 +1,150 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-sentiment" >Sentiment analysis: Plot tool</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Select options</strong>
</div>
<div class="panel-body">
<div class="row">
<!-- left column -->
<div class="col-lg-9">
<!-- providers charts -->
<div class="col-lg-6">
<div class="sparkLineStats">
<ul id="providerList1">
</ul>
</div>
</div>
<div class="col-lg-6">
<div class="sparkLineStats">
<ul id="providerList2">
</ul>
</div>
</div>
</div>
<!-- right column -->
<div class="col-lg-3">
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
</div>
</div>
<!-- /.row -->
<div class="row">
<button id="plot_btn" class="btn btn-info" style="margin-right: 8px; margin-left: 8px;float: right;">Plot!</button>
<label style="float: right;"><input id="checkbox_stacked" type="checkbox" checked=true> Stacked graph?</label>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel PLOT -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph" style="height: 300px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
});
</script>
<script src="{{ url_for('static', filename='js/sentiment_plot.js') }}"></script>
</body>
</html>

View file

@ -0,0 +1,518 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/FlexGauge.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.sparkline.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.canvasjs.min.js') }}"></script>
<style>
.moodtable_worst {
background: rgba(255, 0, 0, 0.47);
font-size: small;
}
.moodtable_best {
background: rgba(132, 255, 0, 0.5);
font-size: small;
}
.jqstooltip{
box-sizing: content-box;
}
strong {
font-size: 16px;
}
.table {
margin-bottom: 0px;
}
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.sparkLineStats {
position: relative;
margin-bottom: -4px;
}
.sparkLineStats ul li {
margin-bottom: 8px;
line-height: 90px;
font-size: 12px;
}
.sparkLineStats ul li div {
float: left;
}
.sparkLineStats ul li div:first-child {
margin-right: 5px;
}
.panelInside {
padding: 5px;
}
.fg-dial-label {
font-size: 100%;
font-weight: bold;
left: 0;
position: relative;
text-align: center;
top: -60px;
margin-bottom: -10px;
width: 100%;
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-sentiment" >Sentiment analysis: Trending</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Pannel TODAY -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong data-toggle="tooltip" data-placement="right" title="Providers displayed are in the top list in Module Statistics">Today's mood</strong>
</div>
<div class="panel-body">
<!-- left column -->
<div class="col-lg-9" style="padding-left: 0px;">
<!-- providers charts -->
<div class="col-lg-6">
<div class="sparkLineStats">
<div id="panel-today" class="panel panel-default pannelToday1">
<div class="panel-heading">
<strong class="sparkLineStatsToday1t">Graph 1</strong>
<strong class="sparkLineStatsToday1s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday2">
<div class="panel-heading">
<strong class="sparkLineStatsToday2t">Graph 2</strong>
<strong class="sparkLineStatsToday2s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday3">
<div class="panel-heading">
<strong class="sparkLineStatsToday3t">Graph 3</strong>
<strong class="sparkLineStatsToday3s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday4">
<div class="panel-heading">
<strong class="sparkLineStatsToday4t">Graph 4</strong>
<strong class="sparkLineStatsToday4s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="col-lg-6">
<div class="sparkLineStats">
<div id="panel-today" class="panel panel-default pannelToday5">
<div class="panel-heading">
<strong class="sparkLineStatsToday5t">Graph 5</strong>
<strong class="sparkLineStatsToday5s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday6">
<div class="panel-heading">
<strong class="sparkLineStatsToday6t">Graph 6</strong>
<strong class="sparkLineStatsToday6s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday7">
<div class="panel-heading">
<strong class="sparkLineStatsToday7t">Graph 7</strong>
<strong class="sparkLineStatsToday7s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-today" class="panel panel-default pannelToday8">
<div class="panel-heading">
<strong class="sparkLineStatsToday8t">Graph 8</strong>
<strong class="sparkLineStatsToday8s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<!-- right column -->
<div class="col-lg-3">
<div class="well text-center" style="padding: 0px;">
<strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the last hour max value">Mood value</strong>
<div id="gauge_today_last_hour"></div>
<strong data-toggle="tooltip" data-placement="top" title="Average of the sentiments' intensity">Compound by mood</strong>
<div id="bar_today_last_hour" style="height: 70px; width: 100%;"></div>
</div>
<div class="well text-center" style="padding: 0px;">
<strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the today max value">Mood value</strong>
<div id="gauge_today_last_days"></div>
<strong data-toggle="tooltip" data-placement="top" title="Average of the of sentiments' intensity">Compound by mood</strong>
<div id="bar_today_last_days" style="height: 70px; width: 100%;"></div>
</div>
</div>
<!-- /.row -->
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Pannel WEEK -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Week's mood</strong>
</div>
<div class="panel-body">
<!-- left column -->
<div class="col-lg-9" style="padding-left: 0px;">
<!-- providers charts -->
<div class="col-lg-6">
<div class="sparkLineStats">
<div id="panel-week" class="panel panel-default pannelWeek1">
<div class="panel-heading">
<strong class="sparkLineStatsWeek1t">Graph 1</strong>
<strong class="sparkLineStatsWeek1s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek2">
<div class="panel-heading">
<strong class="sparkLineStatsWeek2t">Graph 2</strong>
<strong class="sparkLineStatsWeek2s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek3">
<div class="panel-heading">
<strong class="sparkLineStatsWeek3t">Graph 3</strong>
<strong class="sparkLineStatsWeek3s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek4">
<div class="panel-heading">
<strong class="sparkLineStatsWeek4t">Graph 4</strong>
<strong class="sparkLineStatsWeek4s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="col-lg-6">
<div class="sparkLineStats">
<div id="panel-week" class="panel panel-default pannelWeek5">
<div class="panel-heading">
<strong class="sparkLineStatsWeek5t">Graph 5</strong>
<strong class="sparkLineStatsWeek5s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek6">
<div class="panel-heading">
<strong class="sparkLineStatsWeek6t">Graph 6</strong>
<strong class="sparkLineStatsWeek6s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek7">
<div class="panel-heading">
<strong class="sparkLineStatsWeek7t">Graph 7</strong>
<strong class="sparkLineStatsWeek7s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
<div id="panel-week" class="panel panel-default pannelWeek8">
<div class="panel-heading">
<strong class="sparkLineStatsWeek8t">Graph 8</strong>
<strong class="sparkLineStatsWeek8s pull-right">Avg</strong>
</div>
<div class="panel-body panelInside">
<table class="table">
<tbody>
<tr>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8"></div></td>
<td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8b"></div></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<!-- right column -->
<div class="col-lg-3">
<div class="well text-center" style="padding: 0px;">
<strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the week max value">Mood value</strong>
<div id="gauge_week"></div>
</div>
<div class="well text-center" style="padding: 0px;">
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th>Worst mood</th>
<th>Best mood</th>
</tr>
</thead>
<tbody>
<tr>
<td class="worst1 moodtable_worst">worst1</td>
<td class="best1 moodtable_best">best1</td>
</tr>
<tr>
<td class="worst2 moodtable_worst">worst2</td>
<td class="best2 moodtable_best">best2</td>
</tr>
<tr>
<td class="worst3 moodtable_worst">worst3</td>
<td class="best3 moodtable_best">best3</td>
</tr>
<tr>
<td class="worst4 moodtable_worst">worst4</td>
<td class="best4 moodtable_best">best4</td>
</tr>
<tr>
<td class="worst5 moodtable_worst">worst5</td>
<td class="best5 moodtable_best">best5</td>
</tr>
</tbody>
</table>
</div>
</div>
<!-- /.row -->
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script src="{{ url_for('static', filename='js/sentiment_trending.js') }}"></script>
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
// Reload every 30min
setTimeout(function(){ location.reload(); }, 30*60*1000);
});
</script>
</body>
</html>

View file

@ -0,0 +1,351 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.btn-link {
color: #000000
}
.popover-content {
white-space:pre-wrap;
word-wrap:break-word;
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Manage tracked terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Term</th>
<th>Added date</th>
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th># Concerned pastes</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% set i = 0 %}
{% for term in track_list %}
<tr>
<td>{{ term }}</td>
<td>{{ track_list_values[i][3] }}</td>
<td>{{ track_list_values[i][0] }}</td>
<td>{{ track_list_values[i][1] }}</td>
<td>{{ track_list_values[i][2] }}</td>
<td>{{ track_list_num_of_paste[i] }}</td>
<td><p style="margin: 0px;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-danger">
<div class="panel-heading">
<strong>Manage blacklisted terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
<input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable2">
<thead>
<tr>
<th style="max-width: 800px;">Termx</th>
<th>Added date</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% set i = 0 %}
{% for term in black_list %}
<tr>
<td>{{ black_list[i][0] }}</td>
<td>{{ black_list[i][1] }}</td>
<td><p style="margin: 0px;">
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ black_list[i][0] }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
$("#blacklistTermInput").val("");
}
});
perform_binding();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms_management_query_paste') }}?term=" + $(this).attr('data-term');
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Encoding</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th>Mime</th>";
html_to_add += "<th>(# lines, Max line length)</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.encoding+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.mime+"</td>";
html_to_add += "<td>("+curr_data.lineinfo[0]+", "+curr_data.lineinfo[1]+")</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
});
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
});
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val().toLowerCase();;
} else {
var curr_term = $(this).attr('data-content').toLowerCase();;
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
if (curr_term != "") {
console.log(data_to_send);
$.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
$.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
return table_track.cell( rowIdx, 0 ).data() === json.term;
} );
table_track.rows(index).remove().draw( false );
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
$.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
console.log(data2);
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
return table_black.cell( rowIdx, 0 ).data() === json.term;
} );
table_black.rows(index).remove().draw( false );
}
}
}
});
}
}
</script>

View file

@ -0,0 +1,257 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Select options</strong>
</div>
<div class="panel-body">
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
<div class="form-group input-group" style="margin-top: 30px;">
<span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
<input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
<button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
<button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel PLOT -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph" style="height: 300px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
/* Create the slider and button*/
$('#plot-btn-add').hide();
var today = Date.now();
var old_day = today - (12*31*24*60*60)*1000;
$( ".sliderRange" ).slider({
range: true,
min: old_day,
max: today,
values: [ today - (7*24*60*60)*1000, today ],
step: 24*60*60*1000,
slide: function( event, ui ) {
$( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
}
});
$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
" - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
$('#plot-btn').click(plotData);
$('#plot-btn-add').click(addData);
$("#TermInput").val($("#TermInput").attr("data-init-plot"));
if($("#TermInput").attr("data-init-plot") != "") {
$("#plot-btn").click();
}
$("#TermInput").keyup(function(event){
if(event.keyCode == 13){
$("#plot-btn").click();
$("#TermInput").val("");
}
});
});
</script>
<script>
var plot;
var graph_data = [];
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
autoscaleMargin: 0.1,
},
}
function plotData() {
$('#plot-btn-add').show("fast");
var curthis = $(this);
var term = $('#TermInput').val();
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
$.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
graph_data = [];
var to_plot = [];
var curr_data = [];
for(i=0; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
to_plot.push({ data: curr_data, label: term});
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), to_plot, graph_options);
$("#graph").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY-15, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
$("#TermInput").val("");
})
}
function addData() {
var curthis = $(this);
var term = $('#TermInput').val();
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
$.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
var to_plot = [];
var curr_data = [];
for(i=0; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
to_plot.push({ data: curr_data, label: term});
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), graph_data, graph_options);
$("#TermInput").val("");
})
}
</script>
</body>
</html>

View file

@ -0,0 +1,576 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.table > tbody > tr > td {
padding: 5px;
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Today</strong>
</div>
<div class="panel-body">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Today top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-today" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Week</strong>
</div>
<div class="panel-body">
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Week top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-week" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Month</strong>
</div>
<div class="panel-body">
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Month top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-month" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
});
</script>
<script>
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
legend: { show: true,
noColumns: 1,
position: "nw",
labelFormatter: function(label, series) {
return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
}
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
//transform: function (v) { return v < 1 ? v : Math.log(v); }
autoscaleMargin: 0.1,
},
tooltip: true,
tooltipOpts: {
content: " %s (%x.1 is %y.4) ",
shifts: {
x: -60,
y: 25
}
}
}
set_today = "TopTermFreq_set_day";
set_week = "TopTermFreq_set_week";
set_month = "TopTermFreq_set_month";
default_num_curves = 8;
var plot_today;
var plot_week;
var plot_month;
var promises = []; // Used to know when everything has been received
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today, num_day: 5 }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table_today = $("#table-today")
var table_today2 = $("#table-today2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
else
table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=1;
plot_today = $.plot($("#graph-today"), to_plot, graph_options);
hide_unchecked_curves(plot_today, unchecked_label);
$("#graph-today").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, num_day: 7 }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-week")
var table2 = $("#table-week2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=2;
plot_week = $.plot($("#graph-week"), to_plot, graph_options);
hide_unchecked_curves(plot_week, unchecked_label);
$("#graph-week").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month, num_day: 31 }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-month")
var table2 = $("#table-month2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
}
graph_options.series.graphNum=3;
plot_month = $.plot($("#graph-month"), to_plot, graph_options);
hide_unchecked_curves(plot_month, unchecked_label);
$("#graph-month").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
/* When everything has been received, start adding tooltip */
$.when.apply($, promises).done( function (arg) {
$('[data-toggle="tooltip"]').tooltip();
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
$(".check-interaction").unbind("click.interaction");
$(".check-interaction").bind("click.interaction", hide_or_show);
});
function addbuttons(term) {
return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
"data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
"<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
"data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
}
function addcheckbox(graph, term, checked) {
var checked_text = checked ? "checked" : "";
return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_term = $(this).attr('data-term');
var data_to_send = { section: curr_section, action:"add", term: curr_term};
$.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
location.reload();
}
});
}
function hide_unchecked_curves(plot, unchecked_label) {
var graphData = plot.getData();
var index;
for(i=0; i<graphData.length; i++) {
if($.inArray( graphData[i].label, unchecked_label ) != -1){
graphData[i].lines.show = false;
}
}
plot.setData(graphData);
plot.draw();
}
function hide_or_show() {
var curr_term = $(this).attr('data-term');
var graph = $(this).attr('data-graph');
var checked = $(this).prop('checked')
if(graph == "today") {
var graphData = plot_today.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_today.setData(graphData);
plot_today.draw();
} else if (graph == "week") {
var graphData = plot_week.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_week.setData(graphData);
plot_week.draw();
} else if (graph == "month") {
var graphData = plot_month.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_month.setData(graphData);
plot_month.draw();
}
// graph, hide curve
}
function hide_or_show2(index, graphNum) {
if (graphNum == 1)
var plot = plot_today;
else if (graphNum == 2)
var plot = plot_week;
else if (graphNum == 3)
var plot = plot_month;
var graphData = plot.getData();
graphData[index].lines.show = !graphData[index].lines.show;
plot.setData(graphData);
plot.draw();
}
</script>
</body>
</html>

View file

@ -37,6 +37,15 @@ wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTabl
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
#Ressources for sparkline and canvasJS
wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
unzip temp/canvasjs.zip -d temp/
mkdir temp
mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
rm -rf temp
mkdir -p ./static/image mkdir -p ./static/image
pushd static/image pushd static/image