Merge branch 'master' of github.com:CIRCL/AIL-framework

bin/Helper.py

@@ -108,6 +108,7 @@ class Process(object):
         self.modules = ConfigParser.ConfigParser()
         self.modules.read(modulesfile)
         self.subscriber_name = conf_section
+
         self.pubsub = None
         if self.modules.has_section(conf_section):
             self.pubsub = PubSub()
@@ -118,6 +119,15 @@ class Process(object):
             port=self.config.get('RedisPubSub', 'port'),
             db=self.config.get('RedisPubSub', 'db'))
 
+        self.moduleNum = 1
+        for i in range(1, 50):
+            curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i))
+            if curr_num is None:
+                self.moduleNum = i
+                break
+
+
+
     def populate_set_in(self):
         # monoproc
         src = self.modules.get(self.subscriber_name, 'subscribe')
@@ -142,15 +152,18 @@ class Process(object):
 
         else:
             try:
-                path = message.split(".")[-2].split("/")[-1]
+                if ".gz" in message:
+                    path = message.split(".")[-2].split("/")[-1]
+                else:
+                    path = "?"
                 value = str(timestamp) + ", " + path
-                self.r_temp.set("MODULE_"+self.subscriber_name, value)
+                self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
                 return message
 
             except:
                 path = "?"
                 value = str(timestamp) + ", " + path
-                self.r_temp.set("MODULE_"+self.subscriber_name, value)
+                self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
                 return message
 
     def populate_set_out(self, msg, channel=None):

bin/LAUNCH.sh

@@ -112,6 +112,8 @@ function launching_scripts {
 
     echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
+    screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
+    sleep 0.1
     screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
@@ -159,8 +161,6 @@ function launching_scripts {
     screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
-    sleep 0.1
-    screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x'
 
 }
 

bin/ModuleInformation.py

@@ -1,6 +1,18 @@
 #!/usr/bin/env python2
 # -*-coding:UTF-8 -*
 
+'''
+
+This module can be use to see information of running modules.
+These information are logged in "logs/moduleInfo.log"
+
+It can also try to manage them by killing inactive one.
+However, it does not support mutliple occurence of the same module
+(It will kill the first one obtained by get)
+
+
+'''
+
 import time
 import datetime
 import redis
@@ -29,6 +41,9 @@ def getPid(module):
         else:
             return None
 
+def clearRedisModuleInfo():
+    for k in server.keys("MODULE_*"):
+        server.delete(k)
 
 def kill_module(module):
     print ''
@@ -62,6 +77,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
     parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
     parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
+    parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)')
 
     args = parser.parse_args()
 
@@ -80,76 +96,103 @@ if __name__ == "__main__":
         port=cfg.getint("Redis_Queues", "port"),
         db=cfg.getint("Redis_Queues", "db"))
 
-    while True:
+    if args.clear == 1:
-
+        clearRedisModuleInfo()
-        num = 0
-        printarray1 = []
-        printarray2 = []
-        for queue, card in server.hgetall("queues").iteritems():
-            key = "MODULE_" + queue
-            value = server.get(key)
-            if value is not None:
-                timestamp, path = value.split(", ")
-                if timestamp is not None and path is not None:
-                    num += 1
-                    startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
-                    processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
-
-                    if int(card) > 0:
-                        if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
-                            log = open(log_filename, 'a')
-                            log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
-                            if args.autokill == 1:
-                                kill_module(queue)
-
-                        printarray1.append([str(num), str(queue), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
-
-                    else:
-                        printarray2.append([str(num), str(queue), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
-
-        printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
-        printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
-        printarray1.insert(0,["#", "Queue", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
-        printarray2.insert(0,["#", "Queue", "Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
-
-        os.system('clear')
-        t1 = AsciiTable(printarray1, title="Working queues")
-        t1.column_max_width(1)
-        if not t1.ok:
-                longest_col = t1.column_widths.index(max(t1.column_widths))
-                max_length_col = t1.column_max_width(longest_col)
-                if max_length_col > 0:
-                    for i, content in enumerate(t1.table_data):
-                        if len(content[longest_col]) > max_length_col:
-                            temp = ''
-                            for l in content[longest_col].splitlines():
-                                if len(l) > max_length_col:
-                                    temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
-                                else:
-                                    temp += l + '\n'
-                                content[longest_col] = temp.strip()
-                        t1.table_data[i] = content
-
-        t2 = AsciiTable(printarray2, title="Idling queues")
-        t2.column_max_width(1)
-        if not t2.ok:
-                longest_col = t2.column_widths.index(max(t2.column_widths))
-                max_length_col = t2.column_max_width(longest_col)
-                if max_length_col > 0:
-                    for i, content in enumerate(t2.table_data):
-                        if len(content[longest_col]) > max_length_col:
-                            temp = ''
-                            for l in content[longest_col].splitlines():
-                                if len(l) > max_length_col:
-                                    temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
-                                else:
-                                    temp += l + '\n'
-                                content[longest_col] = temp.strip()
-                        t2.table_data[i] = content
 
 
-        print t1.table
+    module_file_array = set()
-        print '\n'
+    with open('../doc/all_modules.txt', 'r') as module_file:
-        print t2.table
+        for line in module_file:
+            module_file_array.add(line[:-1])
 
-        time.sleep(args.refresh)
+        while True:
+
+            all_queue = set()
+            curr_range = 50
+            printarray1 = []
+            printarray2 = []
+            printarray3 = []
+            for queue, card in server.hgetall("queues").iteritems():
+                all_queue.add(queue)
+                key = "MODULE_" + queue + "_"
+                for i in range(1, 50):
+                    curr_num = server.get("MODULE_"+ queue + "_" + str(i))
+                    if curr_num is None:
+                        curr_range = i
+                        break
+
+                for moduleNum in range(1, curr_range):
+                    value = server.get(key + str(moduleNum))
+                    if value is not None:
+                        timestamp, path = value.split(", ")
+                        if timestamp is not None and path is not None:
+                            startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                            processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
+
+                            if int(card) > 0:
+                                if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
+                                    log = open(log_filename, 'a')
+                                    log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
+                                    if args.autokill == 1:
+                                        kill_module(queue)
+
+                                printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
+
+                            else:
+                                printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
+
+            for curr_queue in module_file_array:
+                if curr_queue not in all_queue:
+                    printarray3.append([curr_queue, "Not running"])
+
+            printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
+            printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
+            printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
+            printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
+            printarray3.insert(0,["Queue", "State"])
+
+            os.system('clear')
+            t1 = AsciiTable(printarray1, title="Working queues")
+            t1.column_max_width(1)
+            if not t1.ok:
+                    longest_col = t1.column_widths.index(max(t1.column_widths))
+                    max_length_col = t1.column_max_width(longest_col)
+                    if max_length_col > 0:
+                        for i, content in enumerate(t1.table_data):
+                            if len(content[longest_col]) > max_length_col:
+                                temp = ''
+                                for l in content[longest_col].splitlines():
+                                    if len(l) > max_length_col:
+                                        temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
+                                    else:
+                                        temp += l + '\n'
+                                    content[longest_col] = temp.strip()
+                            t1.table_data[i] = content
+
+            t2 = AsciiTable(printarray2, title="Idling queues")
+            t2.column_max_width(1)
+            if not t2.ok:
+                    longest_col = t2.column_widths.index(max(t2.column_widths))
+                    max_length_col = t2.column_max_width(longest_col)
+                    if max_length_col > 0:
+                        for i, content in enumerate(t2.table_data):
+                            if len(content[longest_col]) > max_length_col:
+                                temp = ''
+                                for l in content[longest_col].splitlines():
+                                    if len(l) > max_length_col:
+                                        temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
+                                    else:
+                                        temp += l + '\n'
+                                    content[longest_col] = temp.strip()
+                            t2.table_data[i] = content
+
+            t3 = AsciiTable(printarray3, title="Not running queues")
+            t3.column_max_width(1)
+
+            print t1.table
+            print '\n'
+            print t2.table
+            print '\n'
+            print t3.table
+
+            time.sleep(args.refresh)

bin/WebStats.py

@@ -66,25 +66,7 @@ def compute_progression(server, field_name, num_day, url_parsed):
         # filter
         if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):
             
-            if server.sismember(redis_progression_name_set, keyword): #if keyword is in the set
+            server.zadd("z_top_progression_"+field_name, float(keyword_increase), keyword)
-                server.hset(redis_progression_name, keyword, keyword_increase) #update its value
-
-            elif (server.scard(redis_progression_name_set) < max_set_cardinality):
-                server.sadd(redis_progression_name_set, keyword)
-
-            else: #not in the set
-                #Check value for all members
-                member_set = []
-                for keyw in server.smembers(redis_progression_name_set):
-                    member_set.append((keyw, int(server.hget(redis_progression_name, keyw))))
-                member_set.sort(key=lambda tup: tup[1])
-                if member_set[0][1] < keyword_increase:
-                    print 'removing', member_set[0][0] + '('+str(member_set[0][1])+')', 'and adding', keyword, str(keyword_increase)
-                    #remove min from set and add the new one
-                    server.srem(redis_progression_name_set, member_set[0][0])
-                    server.sadd(redis_progression_name_set, keyword)
-                    server.hdel(redis_progression_name, member_set[0][0])
-                    server.hset(redis_progression_name, keyword, keyword_increase)
 
 
 if __name__ == '__main__':

bin/launch_scripts.sh

@@ -8,6 +8,8 @@ sleep 0.1
 
 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
+    screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
+    sleep 0.1
     screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
@@ -55,5 +57,3 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
     screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
-    sleep 0.1
-    screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py; read x'

doc/generate_graph_data.py

@@ -32,6 +32,9 @@ with open('../bin/packages/modules.cfg', 'r') as f:
                     continue
 
     output_set_graph = set()
+    with open('all_modules.txt', 'w') as f2:
+        for e in all_modules:
+            f2.write(e+"\n")
 
     for module in modules.keys():
         for stream_in in modules[module]['sub']:

var/www/Flask_server.py

@@ -83,18 +83,27 @@ def get_queues(r):
     # We may want to put the llen in a pipeline to do only one query.
     data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()]
     newData = []
+
+    curr_range = 50
     for queue, card in data:
-        key = "MODULE_" + queue
+        key = "MODULE_" + queue + "_"
-        value = r.get(key)
+        for i in range(1, 50):
-        if value is not None:
+            curr_num = r.get("MODULE_"+ queue + "_" + str(i))
-            timestamp, path = value.split(", ")
+            if curr_num is None:
-            if timestamp is not None:
+                curr_range = i
-                startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                break
-                processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
+
-                seconds = int((datetime.datetime.now() - startTime_readable).total_seconds())
+        for moduleNum in range(1, curr_range):
-                newData.append( (queue, card, seconds) )
+            value = r.get(key + str(moduleNum))
-            else:
+            if value is not None:
-                newData.append( (queue, cards, 0) )
+                timestamp, path = value.split(", ")
+                if timestamp is not None:
+                    startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                    processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
+                    seconds = int((datetime.datetime.now() - startTime_readable).total_seconds())
+                    newData.append( (queue, card, seconds, moduleNum) )
+                else:
+                    newData.append( (queue, cards, 0, moduleNum) )
 
     return newData
 
@@ -103,40 +112,6 @@ def list_len(s):
     return len(s)
 app.jinja_env.filters['list_len'] = list_len
 
-def parseStringToList(the_string):
-    strList = ""
-    elemList = []
-    for c in the_string:
-        if c != ']':
-            if c != '[' and c !=' ' and c != '"':
-                strList += c
-        else:
-            the_list = strList.split(',')
-            if len(the_list) == 3:
-               elemList = elemList + the_list
-            elif len(the_list) == 2:
-               elemList.append(the_list)
-            elif len(the_list) > 1:
-               elemList.append(the_list[1:])
-            strList = ""
-    return elemList
-
-def parseStringToList2(the_string):
-    if the_string == []:
-        return []
-    else:
-        res = []
-        tab_str = the_string.split('], [')
-        tab_str[0] = tab_str[0][1:]+']'
-        tab_str[len(tab_str)-1] = '['+tab_str[len(tab_str)-1][:-1]
-        res.append(parseStringToList(tab_str[0]))
-        for i in range(1, len(tab_str)-2):
-            tab_str[i] = '['+tab_str[i]+']'
-            res.append(parseStringToList(tab_str[i]))
-        if len(tab_str) > 1:
-            res.append(parseStringToList(tab_str[len(tab_str)-1]))
-        return res
-
 
 def showpaste(content_range):
     requested_path = request.args.get('paste', '')
@@ -150,7 +125,7 @@ def showpaste(content_range):
     p_mime = paste.p_mime
     p_lineinfo = paste.get_lines_info()
     p_content = paste.get_p_content().decode('utf-8', 'ignore')
-    p_duplicate_full_list = parseStringToList2(paste._get_p_duplicate())
+    p_duplicate_full_list = json.loads(paste._get_p_duplicate())
     p_duplicate_list = []
     p_simil_list = []
     p_hashtype_list = []
@@ -174,7 +149,7 @@ def showpaste(content_range):
         hash_types = []
         comp_vals = []
         for i in indices:
-            hash_types.append(p_duplicate_full_list[i][0])
+            hash_types.append(p_duplicate_full_list[i][0].encode('utf8'))
             comp_vals.append(p_duplicate_full_list[i][2])
             dup_list_removed.append(i)
 
@@ -281,19 +256,9 @@ def progressionCharts():
         return jsonify(bar_values)
 
     else:
-        redis_progression_name = 'top_progression_'+trending_name
+        redis_progression_name = "z_top_progression_" + trending_name
-        redis_progression_name_set = 'top_progression_'+trending_name+'_set'
+        keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
-
+        return jsonify(keyw_value)
-        # Iterate over element in top_x_set and retreive their value
-        member_set = []
-        for keyw in r_serv_charts.smembers(redis_progression_name_set):
-            keyw_value = r_serv_charts.hget(redis_progression_name, keyw)
-            keyw_value = keyw_value if keyw_value is not None else 0
-            member_set.append((keyw, int(keyw_value)))
-        member_set.sort(key=lambda tup: tup[1], reverse=True)
-        if len(member_set) == 0:
-            member_set.append(("No relevant data", int(100)))
-        return jsonify(member_set)
 
 @app.route("/_moduleCharts", methods=['GET'])
 def modulesCharts():
@@ -472,7 +437,7 @@ def sentiment_analysis_trending():
     return render_template("sentiment_analysis_trending.html")
 
 
-@app.route("/sentiment_analysis_getplotdata/")
+@app.route("/sentiment_analysis_getplotdata/", methods=['GET'])
 def sentiment_analysis_getplotdata():
     # Get the top providers based on number of pastes
     oneHour = 60*60
@@ -481,19 +446,27 @@ def sentiment_analysis_getplotdata():
     dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
     dateStart_timestamp = calendar.timegm(dateStart.timetuple())
 
-    to_return = {}
+    getAllProviders = request.args.get('getProviders')
-    range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
+    provider = request.args.get('provider')
-    # if empty, get yesterday top providers
+    allProvider = request.args.get('all')
-    print 'providers_set_'+ get_date_range(1)[1]
+    if getAllProviders == 'True':
-    range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
+        if allProvider == "True":
-    # if still empty, takes from all providers
+            range_providers = r_serv_charts.smembers('all_provider_set')
-    if range_providers == []:
+            return jsonify(list(range_providers))
-        print 'today provider empty'
+        else:
-        range_providers = r_serv_charts.smembers('all_provider_set')
+            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
+            # if empty, get yesterday top providers
+            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
+            # if still empty, takes from all providers
+            if range_providers == []:
+                print 'today provider empty'
+                range_providers = r_serv_charts.smembers('all_provider_set')
+            return jsonify(range_providers)
 
-    for cur_provider in range_providers:
+    elif provider is not None:
-        print cur_provider
+        to_return = {}
-        cur_provider_name = cur_provider + '_'
+
+        cur_provider_name = provider + '_'
         list_date = {}
         for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
             cur_set_name = cur_provider_name + str(cur_timestamp)
@@ -503,9 +476,10 @@ def sentiment_analysis_getplotdata():
                 cur_value = r_serv_sentiment.get(cur_id)
                 list_value.append(cur_value)
             list_date[cur_timestamp] = list_value
-        to_return[cur_provider] = list_date
+        to_return[provider] = list_date
 
-    return jsonify(to_return)
+        return jsonify(to_return)
+    return "Bad request"
 
 
 

var/www/static/js/indexjavascript.js

@@ -223,9 +223,16 @@ function create_queue_table() {
         var tr = document.createElement('TR')
         for(j = 0; j < 2; j++){
             var td = document.createElement('TD')
-            td.appendChild(document.createTextNode(glob_tabvar.row1[i][j]));
+            var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : "";
+            td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum));
             tr.appendChild(td)
         }
+        // Used to decide the color of the row
+        // We have glob_tabvar.row1[][j] with:
+        // - j=0: ModuleName
+        // - j=1: queueLength
+        // - j=2: LastProcessedPasteTime
+        // - j=3: Number of the module belonging in the same category
         if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2)
             tr.className += " danger";
         else if (parseInt(glob_tabvar.row1[i][2]) > 60*1)

var/www/static/js/sentiment_trending.js

@@ -1,4 +1,42 @@
 
+/* Functions and config */
+function add_new_graph_today(id) {
+    return "<div id=\"panel-today\" class=\"panel panel-default pannelToday"+id+"\">" +
+                "<div class=\"panel-heading\">" +
+                    "<strong class=\"sparkLineStatsToday"+id+"t\">Graph "+id+"</strong>" +
+                    "<strong class=\"sparkLineStatsToday"+id+"s pull-right\">Avg</strong>" +
+                "</div>" +
+                "<div class=\"panel-body panelInside\">" +
+                    "<table class=\"table\">" +
+                        "<tbody>" +
+                            "<tr>" +
+                                "<td style=\"border-top: 0px solid #ddd;\"><div class=\"sparkLineStatsToday"+id+"\"></div></td> " +
+                                "<td style=\"border-top: 0px solid #ddd;\"><div class=\"sparkLineStatsToday"+id+"b\"></div></td> " +
+                            "</tr>" +
+                         "</tbody>" +
+                    "</table>" +
+                "</div>" +
+            "</div>";
+};
+function add_new_graph_week(id) {
+    return "<div id=\"panel-week\" class=\"panel panel-default pannelWeek"+id+"\">" +
+                "<div class=\"panel-heading\">" +
+                    "<strong class=\"sparkLineStatsWeek"+id+"t\">Graph "+id+"</strong>" +
+                    "<strong class=\"sparkLineStatsWeek"+id+"s pull-right\">Avg</strong>" +
+                "</div>" +
+                "<div class=\"panel-body panelInside\">" +
+                    "<table class=\"table\">" +
+                        "<tbody>" +
+                            "<tr>" +
+                                "<td style=\"border-top: 0px solid #ddd;\"><div class=\"sparkLineStatsWeek"+id+"\"></div></td> " +
+                                "<td style=\"border-top: 0px solid #ddd;\"><div class=\"sparkLineStatsWeek"+id+"b\"></div></td> " +
+                            "</tr>" +
+                         "</tbody>" +
+                    "</table>" +
+                "</div>" +
+            "</div>";
+}
+
  function generate_offset_to_time(num){
      var to_ret = {};
      for(i=0; i<=num; i++) {
@@ -15,7 +53,7 @@
      var to_ret = {};
      for(i=day; i>=0; i--){
          for(j=0; j<24; j++){
-             var t1 =now.getDate()-i + ":"; 
+             var t1 =now.getDate()-i + ":";
              var t2 =now.getHours()-(23-j);
              t2 = t2 < 0 ? 24+t2 : t2;
              t2 += "h";
@@ -40,34 +78,79 @@
         barColor: '#00bf5f',
         negBarColor: '#f22929',
         zeroColor: '#ffff00',
-            
+
         tooltipFormat: '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>',
 };
 
 
-$.getJSON("/sentiment_analysis_getplotdata/",
+/* Plot and queries */
-    function(data) {
-        var all_data = [];
-        var plot_data = [];
-        var graph_avg = [];
-        var array_provider = Object.keys(data);
-        var dates_providers = Object.keys(data[array_provider[0]]);
-        var dateStart = parseInt(dates_providers[0]);
-        var oneHour = 60*60;
-        var oneWeek = oneHour*24*7;
 
-        var all_graph_day_sum = 0.0;
+var all_graph_day_sum = 0.0;
-        var all_graph_hour_sum = 0.0;
+var all_graph_hour_sum = 0.0;
-        var all_graph_hour_maxVal = 0.0;
+var all_graph_hour_sum_minus = 0.0;
-        var all_day_avg = 0.0;
+var all_graph_hour_maxVal = 0.0;
-        var all_day_avg_maxVal = 0.0;
+var all_day_avg = 0.0;
+var all_day_avg_maxVal = 0.0;
+var graph_avg = [];
+var all_data = [];
+var provider_already_loaded = [];
+var totNumGraph = 0;
+
+// Query all providers name then launch the query and plot process for each of them.
+// When everything is terminated, plot the widgets (Gauge, canvasJS, table)
+// input: all - set to 'True' if you take all providers
+function draw_page(all) {
+    $.getJSON("/sentiment_analysis_getplotdata/?getProviders=True&all="+all,
+        function(data) {
+            var promises = [];
+
+            var the_length = provider_already_loaded.length == 0 ? 0 : provider_already_loaded.length;
+            for(i=0; i<data.length; i++) {
+                if(provider_already_loaded.indexOf(data[i]) != -1) {
+                    continue;
+                } else {
+                    totNumGraph++;
+                    if(i % 2 == 0) {
+                        $("#today_divl").append(add_new_graph_today(i+the_length+1));
+                        $("#week_divl").append(add_new_graph_week(i+the_length+1));
+                    }
+                    else {
+                        $("#today_divr").append(add_new_graph_today(i+the_length+1));
+                        $("#week_divr").append(add_new_graph_week(i+the_length+1));
+                    }
+                    provider_already_loaded.push(data[i])
+                    promises.push(query_and_plot(data[i], i+the_length));
+
+                }
+            }
+
+            $.when.apply($, promises).done( function (arg) {
+                draw_widgets();
+                $("#LoadAll").show('fast');
+            });
+        }
+    );
+}
+
+
+// Query data and plot it for a given provider
+// input - provider: The povider name to be plotted
+// input - graphNum: The number of the graph (Use to plot on correct div)
+function query_and_plot(provider, graphNum) {
+    var query_plot = $.getJSON("/sentiment_analysis_getplotdata/?provider="+provider,
+        function(data) {
+            var plot_data = [];
+            var array_provider = Object.keys(data);
+            var dates_providers = Object.keys(data[array_provider[0]]);
+            var dateStart = parseInt(dates_providers[0]);
+            var oneHour = 60*60;
+            var oneWeek = oneHour*24*7;
 
-        for (graphNum=0; graphNum<8; graphNum++) {
             var max_value = 0.0;
             var max_value_day = 0.0;
             var graph_data = [];
             var spark_data = [];
-            var curr_provider = array_provider[graphNum];
+            var curr_provider = array_provider[0];
             var curr_sum = 0.0;
             var curr_sum_elem = 0.0;
             var day_sum = 0.0;
@@ -88,7 +171,6 @@ $.getJSON("/sentiment_analysis_getplotdata/",
                     var neu = 0;
 
                     for(i=0; i<data_array.length; i++){
-                        //console.log(data_array[i].replace(/\'/g, '\"'));
                         var curr_data = jQuery.parseJSON(data_array[i].replace(/\'/g, '\"'));
                         compPosAvg += curr_data['compoundPos'];
                         compNegAvg += curr_data['compoundNeg'];
@@ -109,8 +191,7 @@ $.getJSON("/sentiment_analysis_getplotdata/",
                     max_value = Math.abs(pos-neg) > max_value ? Math.abs(pos-neg) : max_value;
 
                     if(curr_date >= dateStart+oneWeek-23*oneHour){
-                        max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day;
+                        max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day; day_sum += (pos-neg);
-                        day_sum += (pos-neg);
                         day_sum_elem++;
                     }
                     if(curr_date > dateStart+oneWeek-2*oneHour && curr_date <=dateStart+oneWeek-oneHour){
@@ -121,17 +202,16 @@ $.getJSON("/sentiment_analysis_getplotdata/",
             }
             all_graph_day_sum += day_sum;
             all_graph_hour_sum += hour_sum;
+            all_graph_hour_sum_minus += hour_sum > 0 ? 0 : 1;
             all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal;
 
-            var curr_avg = curr_sum / (curr_sum_elem); 
+            var curr_avg = curr_sum / (curr_sum_elem);
             if(isNaN(curr_avg))
                 curr_avg = 0.0
-            //var curr_avg = curr_sum / (oneWeek/oneHour); 
-            //var curr_avg = curr_sum / (spark_data.length); 
             graph_avg.push([curr_provider, curr_avg]);
             plot_data.push(spark_data);
             all_data.push(graph_data);
-        
+
 
             sparklineOptions.chartRangeMax = max_value;
             sparklineOptions.chartRangeMin = -max_value;
@@ -141,11 +221,11 @@ $.getJSON("/sentiment_analysis_getplotdata/",
             var num = graphNum + 1;
             var placeholder = '.sparkLineStatsWeek' + num;
             sparklineOptions.barWidth = 2;
-            $(placeholder).sparkline(plot_data[graphNum], sparklineOptions);
+            $(placeholder).sparkline(plot_data[0], sparklineOptions);
             $(placeholder+'t').text(curr_provider);
-            var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); 
+            var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5);
             $(placeholder+'s').text(curr_avg_text);
-        
+
             sparklineOptions.barWidth = 18;
             sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
             $(placeholder+'b').sparkline([curr_avg], sparklineOptions);
@@ -169,8 +249,8 @@ $.getJSON("/sentiment_analysis_getplotdata/",
 
 
             // print today
-            var data_length = plot_data[graphNum].length;
+            var data_length = plot_data[0].length;
-            var data_today = plot_data[graphNum].slice(data_length-24, data_length);
+            var data_today = plot_data[0].slice(data_length-24, data_length);
 
             placeholder = '.sparkLineStatsToday' + num;
             sparklineOptions.barWidth = 14;
@@ -199,155 +279,124 @@ $.getJSON("/sentiment_analysis_getplotdata/",
                 $(avgName).addClass("panel-warning")
             }
 
-        }//for loop
 
-
-
-        /* ---------------- Gauge ---------------- */
-        var gaugeOptions = {
-            animateEasing: true,
-        
-            elementWidth: 200,
-            elementHeight: 125,
-        
-            arcFillStart: 10,
-            arcFillEnd: 12,
-            arcFillTotal: 20,
-            incTot: 1.0,
-        
-            arcBgColorLight: 200,
-            arcBgColorSat: 0,
-            arcStrokeFg: 20,
-            arcStrokeBg: 30,
-        
-            colorArcFg: '#FF3300',
-            animateSpeed: 1,
-        
-        };
-        // Clone object
-        var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions);
-        var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions);
-        
-        
-        
-        gaugeOptions.appendTo = '#gauge_today_last_hour';
-        gaugeOptions.dialLabel = 'Last hour';
-        gaugeOptions.elementId = 'gauge1';
-        var piePercent = (all_graph_hour_sum / 8) / all_graph_hour_maxVal;
-        gaugeOptions.inc = piePercent;
-        var gauge_today_last_hour = new FlexGauge(gaugeOptions);
-        
-        gaugeOptions2.appendTo = '#gauge_today_last_days';
-        gaugeOptions2.dialLabel = 'Today';
-        gaugeOptions2.elementId = 'gauge2';
-        //piePercent = (all_graph_day_sum / (8*24)) / max_value;
-        piePercent = (all_day_avg / 8) / all_day_avg_maxVal;
-        gaugeOptions2.inc = piePercent;
-        var gauge_today_last_days = new FlexGauge(gaugeOptions2);
-        
-        gaugeOptions3.appendTo = '#gauge_week';
-        gaugeOptions3.dialLabel = 'Week';
-        gaugeOptions3.elementId = 'gauge3';
-
-        var graph_avg_sum = 0.0;
-        var temp_max_val = 0.0;
-        for (i=0; i<graph_avg.length; i++){
-            graph_avg_sum += graph_avg[i][1];
-            temp_max_val = Math.abs(graph_avg[i][1]) > temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val;
         }
-
+    );
-        piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val;
+    return query_plot
-        gaugeOptions3.inc = piePercent;
+}
-        var gauge_today_last_days = new FlexGauge(gaugeOptions3);
 
 
-        /* --------- Sort providers -------- */
 
-        graph_avg.sort(function(a, b){return b[1]-a[1]});
+function draw_widgets() {
 
-        for (i=1; i<6; i++){
+    /* ---------------- Gauge ---------------- */
-            $('.worst'+i).text(graph_avg[7-(i-1)][0]);
+    var gaugeOptions = {
-            $('.best'+i).text(graph_avg[i-1][0]);
+        animateEasing: true,
-        }
 
-        /* ----------- CanvasJS ------------ */
+        elementWidth: 200,
+        elementHeight: 125,
 
-        var comp_sum_day_pos = 0.0;
+        arcFillStart: 10,
-        var comp_sum_day_neg = 0.0;
+        arcFillEnd: 12,
-        var comp_sum_hour_pos = 0.0;
+        arcFillTotal: 20,
-        var comp_sum_hour_neg = 0.0;
+        incTot: 1.0,
-        for(graphNum=0; graphNum<8; graphNum++){
-            curr_graphData = all_data[graphNum];
-            var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length);
-            for (i=1; i< gauge_data.length; i++){
-                comp_sum_day_pos += gauge_data[i].compoundPos;
-                comp_sum_day_neg += gauge_data[i].compoundNeg;
 
-                if(i == 23){
+        arcBgColorLight: 200,
-                    comp_sum_hour_pos += gauge_data[i].compoundPos;
+        arcBgColorSat: 0,
-                    comp_sum_hour_neg += gauge_data[i].compoundNeg;
+        arcStrokeFg: 20,
-                }
+        arcStrokeBg: 30,
+
+        colorArcFg: '#FF3300',
+        animateSpeed: 1,
+
+    };
+    // Clone object
+    var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions);
+    var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions);
+
+
+
+    gaugeOptions.appendTo = '#gauge_today_last_hour';
+    gaugeOptions.dialLabel = 'Last hour';
+    gaugeOptions.elementId = 'gauge1';
+    var piePercent = (all_graph_hour_sum / (totNumGraph - all_graph_hour_sum_minus)) / all_graph_hour_maxVal;
+    gaugeOptions.inc = piePercent;
+    var gauge_today_last_hour = new FlexGauge(gaugeOptions);
+
+    gaugeOptions2.appendTo = '#gauge_today_last_days';
+    gaugeOptions2.dialLabel = 'Today';
+    gaugeOptions2.elementId = 'gauge2';
+    piePercent = (all_day_avg / totNumGraph) / all_day_avg_maxVal;
+    gaugeOptions2.inc = piePercent;
+    var gauge_today_last_days = new FlexGauge(gaugeOptions2);
+
+    gaugeOptions3.appendTo = '#gauge_week';
+    gaugeOptions3.dialLabel = 'Week';
+    gaugeOptions3.elementId = 'gauge3';
+
+    var graph_avg_sum = 0.0;
+    var temp_max_val = 0.0;
+    for (i=0; i<graph_avg.length; i++){
+        graph_avg_sum += graph_avg[i][1];
+        temp_max_val = Math.abs(graph_avg[i][1]) > temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val;
+    }
+
+    piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val;
+    gaugeOptions3.inc = piePercent;
+    var gauge_today_last_days = new FlexGauge(gaugeOptions3);
+
+
+    /* --------- Sort providers -------- */
+
+    graph_avg.sort(function(a, b){return b[1]-a[1]});
+
+    for (i=1; i<6; i++){
+        $('.worst'+i).text(graph_avg[7-(i-1)][0]);
+        $('.best'+i).text(graph_avg[i-1][0]);
+    }
+
+    /* ----------- CanvasJS ------------ */
+
+    var comp_sum_day_pos = 0.0;
+    var comp_sum_day_neg = 0.0;
+    var comp_sum_hour_pos = 0.0;
+    var comp_sum_hour_neg = 0.0;
+    for(graphNum=0; graphNum<totNumGraph; graphNum++){
+        curr_graphData = all_data[graphNum];
+        var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length);
+        for (i=1; i< gauge_data.length; i++){
+            comp_sum_day_pos += gauge_data[i].compoundPos;
+            comp_sum_day_neg += gauge_data[i].compoundNeg;
+
+            if(i == 23){
+                comp_sum_hour_pos += gauge_data[i].compoundPos;
+                comp_sum_hour_neg += gauge_data[i].compoundNeg;
             }
-
         }
 
-        var options_canvasJS_1 = {
+    }
-          
-            animationEnabled: true,
-            axisY: {
-                tickThickness: 0,
-                lineThickness: 0,
-                valueFormatString: " ",
-                gridThickness: 0              
-            },
-            axisX: {
-                tickThickness: 0,
-                lineThickness: 0,
-                labelFontSize: 0.1,
-            },
-            data: [
-                {
-                    toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
-                    type: "bar",
-                    color: "green",
-                    dataPoints: [
-                        {y: comp_sum_hour_pos/8}
-                    ]
-                },
-                {
-                    toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
-                    type: "bar",
-                    color: "red",
-                    dataPoints: [
-                        {y: comp_sum_hour_neg/8}
-                    ]
-                }
-            ]
-        };
-        
-        var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1);
 
-        var options_canvasJS_2 = {
+    var options_canvasJS_1 = {
-          
+
-            animationEnabled: true,
+        animationEnabled: true,
-            axisY: {
+        axisY: {
-                tickThickness: 0,
+            tickThickness: 0,
-                lineThickness: 0,
+            lineThickness: 0,
-                valueFormatString: " ",
+            valueFormatString: " ",
-                gridThickness: 0              
+            gridThickness: 0
-            },
+        },
-            axisX: {
+        axisX: {
-                tickThickness: 0,
+            tickThickness: 0,
-                lineThickness: 0,
+            lineThickness: 0,
-                labelFontSize: 0.1,
+            labelFontSize: 0.1,
-            },
+        },
-            data: [
+        data: [
             {
                 toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
                 type: "bar",
                 color: "green",
                 dataPoints: [
-                    {y: comp_sum_day_pos/8}
+                    {y: comp_sum_hour_pos/totNumGraph}
                 ]
             },
             {
@@ -355,32 +404,51 @@ $.getJSON("/sentiment_analysis_getplotdata/",
                 type: "bar",
                 color: "red",
                 dataPoints: [
-                    {y: comp_sum_day_neg/8}
+                    {y: comp_sum_hour_neg/totNumGraph}
                 ]
             }
+        ]
+    };
+
+    var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1);
+
+    var options_canvasJS_2 = {
+
+        animationEnabled: true,
+        axisY: {
+            tickThickness: 0,
+            lineThickness: 0,
+            valueFormatString: " ",
+            gridThickness: 0
+        },
+        axisX: {
+            tickThickness: 0,
+            lineThickness: 0,
+            labelFontSize: 0.1,
+        },
+        data: [
+        {
+            toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
+            type: "bar",
+            color: "green",
+            dataPoints: [
+                {y: comp_sum_day_pos/totNumGraph}
             ]
-        };
+        },
-
+        {
-        var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2);
+            toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
-        
+            type: "bar",
-        chart_canvas1.render();
+            color: "red",
-        chart_canvas2.render();
+            dataPoints: [
-
+                {y: comp_sum_day_neg/totNumGraph}
-
+            ]
-
+        }
-    }
+        ]
-);
+    };
-
-
-
-
-
-
-
-
-
-
-
 
+    var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2);
 
+    chart_canvas1.render();
+    chart_canvas2.render();
 
+}

var/www/templates/index.html

@@ -154,7 +154,6 @@
     </script>
 
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-    <script src="{{ url_for('static', filename='js/plugins/metisMenu/metisMenu.js') }}"></script>
 
 </body>
 

var/www/templates/sentiment_analysis_trending.html

@@ -113,6 +113,7 @@
   </div>
     <!-- /.row -->
     <div class="row">
+        <button id="LoadAll" class="btn btn-info" style="margin: 5px;"><span class="glyphicon glyphicon-download"> </span> Load data from all providers </button>
 
     <!-- Pannel TODAY -->
     <div class="row">
@@ -128,139 +129,13 @@
                             <div class="col-lg-9" style="padding-left: 0px;">
                                 <!-- providers charts -->
                                 <div class="col-lg-6">
-                                   <div class="sparkLineStats">
+                                   <div id="today_divl" class="sparkLineStats">
-                                       <div id="panel-today" class="panel panel-default pannelToday1">
+
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday1t">Graph 1</strong>
-                                               <strong class="sparkLineStatsToday1s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday2">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday2t">Graph 2</strong>
-                                               <strong class="sparkLineStatsToday2s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday3">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday3t">Graph 3</strong>
-                                               <strong class="sparkLineStatsToday3s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday4">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday4t">Graph 4</strong>
-                                               <strong class="sparkLineStatsToday4s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
                                    </div>
                                 </div>
                                 <div class="col-lg-6">
-                                   <div class="sparkLineStats">
+                                   <div id="today_divr" class="sparkLineStats">
-                                       <div id="panel-today" class="panel panel-default pannelToday5">
+
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday5t">Graph 5</strong>
-                                               <strong class="sparkLineStatsToday5s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday6">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday6t">Graph 6</strong>
-                                               <strong class="sparkLineStatsToday6s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday7">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday7t">Graph 7</strong>
-                                               <strong class="sparkLineStatsToday7s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
-                                       <div id="panel-today" class="panel panel-default pannelToday8">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsToday8t">Graph 8</strong>
-                                               <strong class="sparkLineStatsToday8s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                        </div>
                                    </div>
                                 </div> 
                             </div>
@@ -304,139 +179,13 @@
                             <div class="col-lg-9" style="padding-left: 0px;">
                                 <!-- providers charts -->
                                 <div class="col-lg-6">
-                                   <div class="sparkLineStats">
+                                   <div id="week_divl" class="sparkLineStats">
-                                       <div id="panel-week" class="panel panel-default pannelWeek1">
+
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek1t">Graph 1</strong>
-                                               <strong class="sparkLineStatsWeek1s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek2">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek2t">Graph 2</strong>
-                                               <strong class="sparkLineStatsWeek2s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek3">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek3t">Graph 3</strong>
-                                               <strong class="sparkLineStatsWeek3s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek4">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek4t">Graph 4</strong>
-                                               <strong class="sparkLineStatsWeek4s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
                                    </div>
                                 </div>
                                 <div class="col-lg-6">
-                                   <div class="sparkLineStats">
+                                   <div id="week_divr" class="sparkLineStats">
-                                       <div id="panel-week" class="panel panel-default pannelWeek5">
+                                       
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek5t">Graph 5</strong>
-                                               <strong class="sparkLineStatsWeek5s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek6">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek6t">Graph 6</strong>
-                                               <strong class="sparkLineStatsWeek6s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek7">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek7t">Graph 7</strong>
-                                               <strong class="sparkLineStatsWeek7s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
-                                       <div id="panel-week" class="panel panel-default pannelWeek8">
-                                           <div class="panel-heading">
-                                               <strong class="sparkLineStatsWeek8t">Graph 8</strong>
-                                               <strong class="sparkLineStatsWeek8s pull-right">Avg</strong>
-                                           </div>
-                                           <div class="panel-body panelInside">
-                                               <table class="table">
-                                                   <tbody>
-                                                       <tr>
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8"></div></td> 
-                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8b"></div></td> 
-                                                       </tr>
-                                                    </tbody>
-                                               </table>
-                                           </div>
-                                       </div>
                                    </div>
                                 </div> 
                             </div>
@@ -490,8 +239,6 @@
         </div>
     </div>
 
-
-
     
     <!-- /.row -->
     </div>
@@ -502,13 +249,15 @@
 <!-- import graph function -->
 <script src="{{ url_for('static', filename='js/sentiment_trending.js') }}"></script>
 <script>
+    $("#LoadAll").hide();
 
     $(document).ready(function(){
         activePage = $('h1.page-header').attr('data-page');
         $("#"+activePage).addClass("active");
         $('[data-toggle="tooltip"]').tooltip(); 
 
-
+        $("#LoadAll").click(function(){ draw_page("True"); });
+        draw_page("False");
         // Reload every 30min
         setTimeout(function(){ location.reload(); }, 30*60*1000);
    });

var/www/templates/terms_management.html

@@ -233,15 +233,15 @@
             $.getJSON(url, function (data) {
                 if (data.length != 0) {
                     var html_to_add = "";
-                    html_to_add += "<table class=\"table table-striped\">";
+                    html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
                     html_to_add += "<thead>";
                       html_to_add += "<tr>";
                         html_to_add += "<th>Source</th>";
                         html_to_add += "<th>Date</th>";
                         html_to_add += "<th>Encoding</th>";
                         html_to_add += "<th>Size (Kb)</th>";
-                        html_to_add += "<th>Mime</th>";
+                        html_to_add += "<th># lines</th>";
-                        html_to_add += "<th>(# lines, Max line length)</th>";
+                        html_to_add += "<th>Max length</th>";
                         html_to_add += "<th>Preview</th>";
                       html_to_add += "</tr>";
                     html_to_add += "</thead>";
@@ -253,8 +253,8 @@
                                 html_to_add += "<td>"+curr_data.date+"</td>";
                                 html_to_add += "<td>"+curr_data.encoding+"</td>";
                                 html_to_add += "<td>"+curr_data.size+"</td>";
-                                html_to_add += "<td>"+curr_data.mime+"</td>";
+                                html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
-                                html_to_add += "<td>("+curr_data.lineinfo[0]+", "+curr_data.lineinfo[1]+")</td>";
+                                html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
                                 html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
 
                             html_to_add += "</tr>";
@@ -264,6 +264,7 @@
                     $("#mymodalbody").html(html_to_add);
                     $("[data-toggle=popover]").popover();
                     $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
+	            $('#modal-table').DataTable();
                 } else {
                     $("#mymodalbody").html("No paste containing this term has been received yet.");
                     $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );