Merge pull request #57 from mokaddem/master

New web pages and statistics modules

.gitignore

@@ -1,6 +1,7 @@
 # Temp files
 *.swp
 *.pyc
+*.swo
 
 # Install Dirs
 AILENV

bin/Url.py

@@ -19,11 +19,11 @@ import ipaddress
 from Helper import Process
 
 # Used to prevent concat with empty fields due to url parsing
-def avoidNone(str):
-    if str is None:
+def avoidNone(a_string):
+    if a_string is None:
         return ""
     else:
-        return str
+        return a_string
 
 if __name__ == "__main__":
     publisher.port = 6380

bin/WebStats.py

@@ -10,19 +10,72 @@ import re
 import redis
 import os
 from packages import lib_words
+from packages.Date import Date
 from pubsublogger import publisher
 from packages import Paste
 from Helper import Process
 from pyfaup.faup import Faup
 
-def analyse(field_name):
+# Config Var
+threshold_need_to_look	= 50
+range_to_look		= 10
+threshold_to_plot	= 1 #500%
+to_plot			= set() 
+clean_frequency		= 10 #minutes
+
+def analyse(server, field_name):
     field = url_parsed[field_name]
     if field is not None:
-        prev_score = r_serv1.hget(field, date)
+        prev_score = server.hget(field, date)
         if prev_score is not None:
-            r_serv1.hset(field, date, int(prev_score) + 1)
+            server.hset(field, date, int(prev_score) + 1)
         else:
-            r_serv1.hset(field, date, 1)
+            server.hset(field, date, 1)
+
+def analyse_and_progression(server, field_name):
+    field = url_parsed[field_name]
+    if field is not None:
+        prev_score = server.hget(field, date)
+        if prev_score is not None:
+            print field + ' prev_score:' + prev_score
+            server.hset(field, date, int(prev_score) + 1)
+            if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive
+                if(check_for_progression(server, field, date)):
+                    to_plot.add(field)
+        else:
+            server.hset(field, date, 1)
+
+def check_for_progression(server, field, date):
+    previous_data = set()
+    tot_sum = 0
+    for i in range(0, range_to_look):
+        curr_value = server.hget(field, Date(date).substract_day(i))
+        if curr_value is None: #no further data
+            break
+        else:
+            curr_value = int(curr_value)
+            previous_data.add(curr_value)
+            tot_sum += curr_value 
+            if i == 0:
+                today_val = curr_value
+
+   
+    print 'totsum='+str(tot_sum)
+    print 'div='+str(tot_sum/today_val)
+    if tot_sum/today_val >= threshold_to_plot:
+        return True
+    else:
+        return False
+
+def clean_to_plot():
+    temp_to_plot = set()
+    curr_date = datetime.date.today()
+    date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day))
+
+    for elem in to_plot:
+        if(check_for_progression(field, date)):
+            temp_to_plot.add(elem)    
+    to_plot = temp_to_plot
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@@ -45,6 +98,11 @@ if __name__ == '__main__':
         host=p.config.get("Redis_Level_DB", "host"),
         port=p.config.get("Redis_Level_DB", "port"),
         db=p.config.get("Redis_Level_DB", "db"))
+ 
+    r_serv2 = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_Domain", "host"),
+        port=p.config.get("Redis_Level_DB_Domain", "port"),
+        db=p.config.get("Redis_Level_DB_Domain", "db"))
 
     # FILE CURVE SECTION #
     csv_path_proto = os.path.join(os.environ['AIL_HOME'],
@@ -57,6 +115,10 @@ if __name__ == '__main__':
     tldsfile_path = os.path.join(os.environ['AIL_HOME'],
                                  p.config.get("Directories", "tldsfile"))
 
+    csv_path_domain = os.path.join(os.environ['AIL_HOME'],
+                            p.config.get("Directories", "domainstrending_csv"))
+
+
     faup = Faup()
     generate_new_graph = False
     # Endless loop getting messages from the input queue
@@ -71,7 +133,7 @@ if __name__ == '__main__':
                 today = datetime.date.today()
                 year = today.year
                 month = today.month
-                
+
                 lib_words.create_curve_with_word_file(r_serv1, csv_path_proto,
                                                       protocolsfile_path, year,
                                                       month)
@@ -80,8 +142,14 @@ if __name__ == '__main__':
                                                       tldsfile_path, year,
                                                       month)
 
+                lib_words.create_curve_with_list(r_serv2, csv_path_domain,
+                                                      to_plot, year,
+                                                      month)
+                print 'end building'
+
             publisher.debug("{} queue is empty, waiting".format(config_section))
-            time.sleep(1)
+            print 'sleeping'
+            time.sleep(5)
             continue
 
 	else:
@@ -91,5 +159,6 @@ if __name__ == '__main__':
             faup.decode(url)
             url_parsed = faup.get()
             
-            analyse('scheme')	#Scheme analysis
-            analyse('tld')	#Tld analysis
+            analyse(r_serv1, 'scheme')	#Scheme analysis
+            analyse(r_serv1, 'tld')	#Tld analysis
+	    analyse_and_progression(r_serv2, 'domain')	#Domain analysis

bin/packages/Date.py

@@ -30,3 +30,12 @@ class Date(object):
 
     def _set_day(self, day):
         self.day = day
+	
+    def substract_day(self, numDay):
+	import datetime
+	computed_date = datetime.date(int(self.year), int(self.month), int(self.day)) - datetime.timedelta(numDay)
+	comp_year = str(computed_date.year)
+        comp_month = str(computed_date.month).zfill(2)
+        comp_day = str(computed_date.day).zfill(2)
+	return comp_year + comp_month + comp_day
+

bin/packages/Paste.py

@@ -186,7 +186,9 @@ class Paste(object):
         if the paste doesn't contain any human dictionnary words
         ..seealso: git@github.com:saffsd/langid.py.git
 
-        """
+        FIXME: This procedure is using more than 20% of CPU
+
+	"""
         identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
         return identifier.classify(self.get_p_content())
 
@@ -196,6 +198,9 @@ class Paste(object):
     def _get_p_date(self):
         return self.p_date
 
+    def _get_p_size(self):
+        return self.p_size
+
     def _get_hash_lines(self, min=1, start=1, jump=10):
         """
         Returning all the lines of the paste hashed.

bin/packages/lib_words.py

@@ -81,13 +81,14 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
     to keep the timeline of the curve correct.
 
     """
+    threshold = 50
     first_day = date(year, month, 01)
     last_day = date(year, month, calendar.monthrange(year, month)[1])
     words = []
 
     with open(feederfilename, 'rb') as f:
         # words of the files
-        words = sorted([word.strip() for word in f])
+        words = sorted([word.strip() for word in f if word.strip()[0:2]!='//' ])
 
     headers = ['Date'] + words
     with open(csvfilename+'.csv', 'wb') as f:
@@ -102,6 +103,47 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
             # from the 1srt day to the last of the list
             for word in words:
                 value = r_serv.hget(word, curdate)
+                if value is None:
+                    row.append(0)
+                else:
+                    # if the word have a value for the day
+                    # FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold
+                    if value >= threshold:
+                        row.append(value)
+            writer.writerow(row)
+
+def create_curve_with_list(server, csvfilename, to_plot, year, month):
+    """Create a csv file used with dygraph.
+
+    :param r_serv: -- connexion to redis database
+    :param csvfilename: -- the path to the .csv file created
+    :param to_plot: -- the list which contain a words to plot.
+    :param year: -- (integer) The year to process
+    :param month: -- (integer) The month to process
+
+    This function create a .csv file using datas in redis.
+    It's checking if the words contained in to_plot and
+    their respectives values by days exists.
+
+    """
+
+    first_day = date(year, month, 01)
+    last_day = date(year, month, calendar.monthrange(year, month)[1])
+    words = sorted(to_plot)
+
+    headers = ['Date'] + words
+    with open(csvfilename+'.csv', 'wb') as f:
+        writer = csv.writer(f)
+        writer.writerow(headers)
+
+        # for each days
+        for dt in rrule(DAILY, dtstart=first_day, until=last_day):
+            row = []
+            curdate = dt.strftime("%Y%m%d")
+            row.append(curdate)
+            # from the 1srt day to the last of the list
+            for word in words:
+                value = server.hget(word, curdate)
                 if value is None:
                     row.append(0)
                 else:

var/www/Flask_server.py

@@ -7,7 +7,9 @@ import json
 from flask import Flask, render_template, jsonify, request
 import flask
 import os
-
+import sys
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Paste
 
 # CONFIG #
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
@@ -19,6 +21,9 @@ if not os.path.exists(configfile):
 cfg = ConfigParser.ConfigParser()
 cfg.read(configfile)
 
+max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
+max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
+
 # REDIS #
 r_serv = redis.StrictRedis(
     host=cfg.get("Redis_Queues", "host"),
@@ -49,6 +54,29 @@ def get_queues(r):
             r.hgetall("queues").iteritems()]
 
 
+def list_len(s):
+    return len(s)
+app.jinja_env.filters['list_len'] = list_len
+
+
+def showpaste(content_range):    
+    requested_path = request.args.get('paste', '')
+    paste = Paste.Paste(requested_path)
+    p_date = str(paste._get_p_date())
+    p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
+    p_source = paste.p_source
+    p_encoding = paste._get_p_encoding()
+    p_language = paste._get_p_language()
+    p_size = paste.p_size
+    p_mime = paste.p_mime
+    p_lineinfo = paste.get_lines_info()
+    p_content = paste.get_p_content().decode('utf-8', 'ignore')
+    if content_range != 0:
+       p_content = p_content[0:content_range] 
+
+    return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content))
+
+
 @app.route("/_logs")
 def logs():
     return flask.Response(event_stream(), mimetype="text/event-stream")
@@ -64,7 +92,10 @@ def search():
     query = request.form['query']
     q = []
     q.append(query)
-    r = []
+    r = [] #complete path
+    c = [] #preview of the paste content
+    paste_date = []
+    paste_size = []
     # Search
     from whoosh import index
     from whoosh.fields import Schema, TEXT, ID
@@ -78,7 +109,16 @@ def search():
         results = searcher.search(query, limit=None)
         for x in results:
             r.append(x.items()[0][1])
-    return render_template("search.html", r=r)
+            paste = Paste.Paste(x.items()[0][1])
+            content = paste.get_p_content().decode('utf8', 'ignore')
+            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+            c.append(content[0:content_range]) 
+            curr_date = str(paste._get_p_date())
+            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+            paste_date.append(curr_date) 
+            paste_size.append(paste._get_p_size()) 
+    return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
+
 
 @app.route("/")
 def index():
@@ -93,16 +133,39 @@ def monitoring():
 
 @app.route("/wordstrending/")
 def wordstrending():
-    return render_template("Wordstrending.html")
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Wordstrending.html", default_display = default_display)
 
 
 @app.route("/protocolstrending/")
 def protocolstrending():
-    return render_template("Protocolstrending.html")
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Protocolstrending.html", default_display = default_display)
 
-@app.route("/tldstrending/")
-def tldstrending():
-    return render_template("Tldstrending.html")
+
+@app.route("/trending/")
+def trending():
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Trending.html", default_display = default_display)
+
+
+@app.route("/showsavedpaste/") #completely shows the paste in a new tab
+def showsavedpaste():
+    return showpaste(0)
+
+
+@app.route("/showpreviewpaste/")
+def showpreviewpaste():
+    return showpaste(max_preview_modal)
+
+
+@app.route("/getmoredata/")
+def getmoredata():
+    requested_path = request.args.get('paste', '')
+    paste = Paste.Paste(requested_path)
+    p_content = paste.get_p_content().decode('utf-8', 'ignore')
+    to_return = p_content[max_preview_modal-1:]
+    return to_return 
 
 
 if __name__ == "__main__":

var/www/static/js/plot-graph.js

@@ -0,0 +1,141 @@
+function Graph(id_pannel, path, header_size){
+	this.path = path;
+	this.id_pannel = id_pannel;
+
+        // Hide every header during initialisation
+        var false_tab = [];
+        for(i=0; i<header_size; i++){
+            false_tab[i] = false;
+        }
+        this.false_tab = false_tab;
+
+        g2 = new Dygraph(
+	    document.getElementById(this.id_pannel),
+	    // path to CSV file
+	    //"{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
+	    //"../static//csv/tldstrendingdata.csv",
+	    this.path,
+	    //window.csv,
+	    {
+	    rollPeriod: 1,
+	    showRoller: true,
+	    //drawPoints: true,
+	    //fillGraph: true,
+	    logscale: true,
+            
+	    animatedZooms: true,
+	    labelsKMB: true,
+	    highlightCircleSize: 3,
+	    highlightSeriesOpts: {
+		strokeWidth: 3,
+		strokeBorderWidth: 1,
+		highlightCircleSize: 5,
+	    },
+	    underlayCallback: function(canvas, area, g) {
+	    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
+
+	    function highlight_period(x_start, x_end) {
+		var canvas_left_x = g.toDomXCoord(x_start);
+		var canvas_right_x = g.toDomXCoord(x_end);
+		var canvas_width = canvas_right_x - canvas_left_x;
+		canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
+	    }
+
+	    var min_data_x = g.getValue(0,0);
+	    var max_data_x = g.getValue(g.numRows()-1,0);
+
+	    // get day of week
+	    var d = new Date(min_data_x);
+	    var dow = d.getUTCDay();
+	    var ds = d.toUTCString();
+
+	    var w = min_data_x;
+	    // starting on Sunday is a special case
+	    if (dow == 0) {
+		highlight_period(w,w+12*3600*1000);
+	    }
+	    // find first saturday
+	    while (dow != 5) {
+		w += 24*3600*1000;
+		d = new Date(w);
+		dow = d.getUTCDay();
+	    }
+
+	    // shift back 1/2 day to center highlight around the point for the day
+	    w -= 12*3600*1000;
+	    while (w < max_data_x) {
+		var start_x_highlight = w;
+		var end_x_highlight = w + 2*24*3600*1000;
+		// make sure we don't try to plot outside the graph
+		if (start_x_highlight < min_data_x) {
+		    start_x_highlight = min_data_x;
+		}
+		if (end_x_highlight > max_data_x) {
+		    end_x_highlight = max_data_x;
+		}
+		highlight_period(start_x_highlight,end_x_highlight);
+		// calculate start of highlight for next Saturday
+		w += 7*24*3600*1000;
+	    }
+	    },
+            visibility: false_tab
+	});
+        this.graph = g2;
+        this.set_Visibility = setVis;
+        this.set_Visibility_andHide = setVis_andHide;
+
+	onclick = function(ev) {
+	    if (g2.isSeriesLocked()) {
+		g2.clearSelection();
+	    }
+	    else {
+		g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
+	    }
+	};
+	g2.updateOptions({clickCallback: onclick}, true);
+
+	var linear = document.getElementById("linear");
+	var log = document.getElementById("log");
+	linear.onclick = function() { setLog(false); }
+	log.onclick = function() { setLog(true); }
+	var setLog = function(val) {
+	    g2.updateOptions({ logscale: val });
+	    linear.disabled = !val;
+	    log.disabled = val;
+	    }
+	function unzoomGraph() {
+	    g2.updateOptions({
+		dateWindow:null,
+		valueRange:null
+	    });
+	}
+
+        // display the top headers
+        function setVis(max_display){
+            headings = this.graph.getLabels();
+            headings.splice(0,1);
+            var sorted_list = new Array();
+            today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution
+            for( i=0; i<headings.length; i++){
+                the_heading = headings[i];
+                //console.log('heading='+the_heading+' tab['+(today-1)+']['+(parseInt(i)+1)+']='+g.getValue(today-1, parseInt(i)+1));
+                sorted_list.push({dom: the_heading, val: this.graph.getValue(today-1, parseInt(i)+1), index: parseInt(i)});
+            }
+            sorted_list.sort(function(a,b) {
+                    return b.val - a.val;
+            });
+
+            var display_list = sorted_list.slice(0, max_display);
+            for( i=0; i<display_list.length; i++){
+                this.graph.setVisibility(display_list[i].index, true);
+            }
+            return sorted_list.slice(0, sorted_list.length);
+       }
+       function setVis_andHide(max_display, old_display){
+           display_list = this.set_Visibility(max_display);
+           for(i=max_display; i<old_display; i++) {
+               this.graph.setVisibility(display_list[i].index, false);
+           } 
+       }
+
+}

var/www/templates/Protocolstrending.html

@@ -16,15 +16,19 @@
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-
+  <script> 
+      var default_display = {{ default_display }};
+      var current_displayed_graph;
+  </script>
 </head>
+
 <body>
 
     <div id="wrapper">
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
           <ul class="nav navbar-nav">
-            <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+            <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li></ul>
          </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
@@ -81,6 +85,13 @@
                                     </ul>
                                 </div>
                             </div>
+
+                            <div class="btn-group btn-group-xs pull-right" style="margin-right: 5px;">
+                                <button type="button" class="btn btn-primary" onclick="take_top(5);">5</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(10);">10</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(15);">15</button>
+                            </div>
+
                         </div>
                         <!-- /.panel-heading -->
                           <div class="panel-body">
@@ -93,102 +104,44 @@
             <!-- /.row -->
         </div>
         <!-- /#page-wrapper -->
-            <script type="text/javascript">
-g2 = new Dygraph(
-    document.getElementById("ProtocolsTrending"),
-    // path to CSV file
-    "{{ url_for('static', filename='csv/protocolstrendingdata.csv') }}",
-    //"../csv/wordstrendingdata.csv",
-    //window.csv,
-    {
-    rollPeriod: 1,
-    showRoller: true,
-    //drawPoints: true,
-    //fillGraph: true,
-    logscale: true,
-    animatedZooms: true,
-    labelsKMB: true,
-    highlightCircleSize: 3,
-    highlightSeriesOpts: {
-        strokeWidth: 3,
-        strokeBorderWidth: 1,
-        highlightCircleSize: 5,
-    },
-    underlayCallback: function(canvas, area, g) {
-    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
 
-    function highlight_period(x_start, x_end) {
-        var canvas_left_x = g.toDomXCoord(x_start);
-        var canvas_right_x = g.toDomXCoord(x_end);
-        var canvas_width = canvas_right_x - canvas_left_x;
-        canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
-    }
+        <!-- import graph function -->
+        <script src="{{ url_for('static', filename='js/plot-graph.js') }}"></script>
 
-    var min_data_x = g.getValue(0,0);
-    var max_data_x = g.getValue(g.numRows()-1,0);
+        <script type="text/javascript">
+            // Create, plot and set the limit of displayed headers
+            function create_and_plot(pannel, path){
+                //var graph_domain = new Graph($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
+                $.get(path, function(myContentFile) {
+                    var lines = myContentFile.split("\r\n");
+                    var header_size = lines[0].split(',').length-1;
+                    current_displayed_graph = new Graph(pannel, path, header_size);
+                    setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300);
+                }, 'text');
+                
+            }
 
-    // get day of week
-    var d = new Date(min_data_x);
-    var dow = d.getUTCDay();
-    var ds = d.toUTCString();
+            // When a pannel is shown, create_and_plot.
+            $('.nav-tabs a').on('shown.bs.tab', function(event){
+                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));    
+            });
+        </script>
 
-    var w = min_data_x;
-    // starting on Sunday is a special case
-    if (dow == 0) {
-        highlight_period(w,w+12*3600*1000);
-    }
-    // find first saturday
-    while (dow != 5) {
-        w += 24*3600*1000;
-        d = new Date(w);
-        dow = d.getUTCDay();
-    }
+        <script>
+            $(document).ready(function(){
+                // Create the graph when the page has just loaded
+                create_and_plot("ProtocolsTrending", "../static//csv/protocolstrendingdata.csv") 
+            });
+ 
+            function take_top(new_display){
+                current_displayed_graph.set_Visibility_andHide(new_display, default_display);
+                default_display = new_display;
+            }
+            
+        </script>
 
-    // shift back 1/2 day to center highlight around the point for the day
-    w -= 12*3600*1000;
-    while (w < max_data_x) {
-        var start_x_highlight = w;
-        var end_x_highlight = w + 2*24*3600*1000;
-        // make sure we don't try to plot outside the graph
-        if (start_x_highlight < min_data_x) {
-            start_x_highlight = min_data_x;
-        }
-        if (end_x_highlight > max_data_x) {
-            end_x_highlight = max_data_x;
-        }
-        highlight_period(start_x_highlight,end_x_highlight);
-        // calculate start of highlight for next Saturday
-        w += 7*24*3600*1000;
-    }
-    }
-});
 
-onclick = function(ev) {
-    if (g2.isSeriesLocked()) {
-        g2.clearSelection();
-    }
-    else {
-        g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
-    }
-};
-g2.updateOptions({clickCallback: onclick}, true);
 
-var linear = document.getElementById("linear");
-var log = document.getElementById("log");
-linear.onclick = function() { setLog(false); }
-log.onclick = function() { setLog(true); }
-var setLog = function(val) {
-    g2.updateOptions({ logscale: val });
-    linear.disabled = !val;
-    log.disabled = val;
-    }
-function unzoomGraph() {
-    g2.updateOptions({
-        dateWindow:null,
-        valueRange:null
-    });
-}
-</script>
     </div>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
 </body>

var/www/templates/Tldstrending.html

@@ -1,196 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-  <title>Analysis Information Leak framework Dashboard</title>
-
-  <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
-  <!-- JS -->
-  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
-  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-
-</head>
-<body>
-
-    <div id="wrapper">
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-          <div class="navbar-header">
-          <ul class="nav navbar-nav">
-              <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
-          </div>
-          <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          <div class="input-group custom-search-form">
-                              <input type="text" class="form-control" placeholder="Search Paste">
-                              <span class="input-group-btn">
-                              <button class="btn btn-default" type="button">
-                                  <i class="fa fa-search"></i>
-                              </button>
-                          </span>
-                          </div>
-                          <!-- /input-group -->
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-        <div id="page-wrapper">
-          <div class="row">
-              <div class="col-lg-12">
-                  <h1 class="page-header">Top Level Domain Trending</h1>
-              </div>
-              <!-- /.col-lg-12 -->
-          </div>
-            <!-- /.row -->
-            <div class="row">
-                <div class="col-lg-12">
-                    <div class="panel panel-default">
-                        <div class="panel-heading">
-                            <i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
-                            <div class="pull-right">
-                                <div class="btn-group">
-                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
-                                        Actions
-                                        <span class="caret"></span>
-                                    </button>
-                                    <ul class="dropdown-menu pull-right" role="menu">
-                                        <li><a href="#" id="linear">Linear Scale</a>
-                                        </li>
-                                        <li><a href="#" id="log">Log Scale</a>
-                                        </li>
-                                        <li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
-                                        </li>
-                                        <li class="divider"></li>
-                                        <li><a href="#" id="edit_graph">Edit graph words</a>
-                                        </li>
-                                    </ul>
-                                </div>
-                            </div>
-                        </div>
-                        <!-- /.panel-heading -->
-                          <div class="panel-body">
-                             <!-- <div id="WordTrending" style="width:100%;"></div> -->
-                              <div id="TldsTrending" style="width:100%; height:800px;"></div>
-                          </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-            <!-- /.row -->
-        </div>
-        <!-- /#page-wrapper -->
-            <script type="text/javascript">
-g2 = new Dygraph(
-    document.getElementById("TldsTrending"),
-    // path to CSV file
-    "{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
-    //"../csv/wordstrendingdata.csv",
-    //window.csv,
-    {
-    rollPeriod: 1,
-    showRoller: true,
-    //drawPoints: true,
-    //fillGraph: true,
-    logscale: true,
-    animatedZooms: true,
-    labelsKMB: true,
-    highlightCircleSize: 3,
-    highlightSeriesOpts: {
-        strokeWidth: 3,
-        strokeBorderWidth: 1,
-        highlightCircleSize: 5,
-    },
-    underlayCallback: function(canvas, area, g) {
-    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
-
-    function highlight_period(x_start, x_end) {
-        var canvas_left_x = g.toDomXCoord(x_start);
-        var canvas_right_x = g.toDomXCoord(x_end);
-        var canvas_width = canvas_right_x - canvas_left_x;
-        canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
-    }
-
-    var min_data_x = g.getValue(0,0);
-    var max_data_x = g.getValue(g.numRows()-1,0);
-
-    // get day of week
-    var d = new Date(min_data_x);
-    var dow = d.getUTCDay();
-    var ds = d.toUTCString();
-
-    var w = min_data_x;
-    // starting on Sunday is a special case
-    if (dow == 0) {
-        highlight_period(w,w+12*3600*1000);
-    }
-    // find first saturday
-    while (dow != 5) {
-        w += 24*3600*1000;
-        d = new Date(w);
-        dow = d.getUTCDay();
-    }
-
-    // shift back 1/2 day to center highlight around the point for the day
-    w -= 12*3600*1000;
-    while (w < max_data_x) {
-        var start_x_highlight = w;
-        var end_x_highlight = w + 2*24*3600*1000;
-        // make sure we don't try to plot outside the graph
-        if (start_x_highlight < min_data_x) {
-            start_x_highlight = min_data_x;
-        }
-        if (end_x_highlight > max_data_x) {
-            end_x_highlight = max_data_x;
-        }
-        highlight_period(start_x_highlight,end_x_highlight);
-        // calculate start of highlight for next Saturday
-        w += 7*24*3600*1000;
-    }
-    }
-});
-
-onclick = function(ev) {
-    if (g2.isSeriesLocked()) {
-        g2.clearSelection();
-    }
-    else {
-        g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
-    }
-};
-g2.updateOptions({clickCallback: onclick}, true);
-
-var linear = document.getElementById("linear");
-var log = document.getElementById("log");
-linear.onclick = function() { setLog(false); }
-log.onclick = function() { setLog(true); }
-var setLog = function(val) {
-    g2.updateOptions({ logscale: val });
-    linear.disabled = !val;
-    log.disabled = val;
-    }
-function unzoomGraph() {
-    g2.updateOptions({
-        dateWindow:null,
-        valueRange:null
-    });
-}
-</script>
-    </div>
-    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-</body>
-
-</html>

var/www/templates/Trending.html

@@ -0,0 +1,198 @@
+	<!DOCTYPE html>
+	<html>
+
+	<head>
+	  <meta charset="utf-8">
+	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+	  <title>Analysis Information Leak framework Dashboard</title>
+
+	  <!-- Core CSS -->
+	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+	  <!-- JS -->
+	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+	  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
+	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+          <script> 
+              var default_display = {{ default_display }};
+              var current_displayed_graph;
+          </script>
+
+	</head>
+	<body>
+
+	    <div id="wrapper">
+	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+		  <div class="navbar-header">
+		  <ul class="nav navbar-nav">
+		      <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li></ul>
+          </div>
+          <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          <div class="input-group custom-search-form">
+                              <input type="text" class="form-control" placeholder="Search Paste">
+                              <span class="input-group-btn">
+                              <button class="btn btn-default" type="button">
+                                  <i class="fa fa-search"></i>
+                              </button>
+                          </span>
+                          </div>
+                          <!-- /input-group -->
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+        <div id="page-wrapper">
+          <div class="row">
+              <div class="col-lg-12">
+                  <h1 class="page-header">Trending charts</h1>
+              </div>
+              <!-- /.col-lg-12 -->
+          </div>
+            <!-- /.row -->
+            <div class="row">
+            
+            <!-- /.nav-tabs -->
+            <ul class="nav nav-tabs">
+                <li class="active"><a data-toggle="tab" href="#tld-tab" data-pannel="TldsTrending" data-path="../static//csv/tldstrendingdata.csv">Top level domains</a></li>
+                <li><a data-toggle="tab" href="#domain-tab" data-pannel="DomainTrending" data-path="../static//csv/domainstrendingdata.csv">Domains</a></li>
+            </ul>
+            </br>
+              
+            <div class="tab-content">  
+                <div class="col-lg-12 tab-pane fade in active" id="tld-tab" >
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
+
+                            <div class="pull-right">
+                                <div class="btn-group">
+                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
+                                        Actions
+                                        <span class="caret"></span>
+                                    </button>
+                                    <ul class="dropdown-menu pull-right" role="menu">
+                                        <li><a href="#" id="linear">Linear Scale</a>
+                                        </li>
+                                        <li><a href="#" id="log">Log Scale</a>
+                                        </li>
+                                        <li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
+                                        </li>
+                                        <li class="divider"></li>
+                                        <li><a href="#" id="edit_graph">Edit graph words</a>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div> 
+                           <div class="btn-group btn-group-xs pull-right" style="margin-right: 5px;">
+                                <button type="button" class="btn btn-primary" onclick="take_top(5);">5</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(10);">10</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(15);">15</button>
+                            </div>
+
+                        </div>
+                        <!-- /.panel-heading -->
+                          <div class="panel-body">
+                             <!-- <div id="TldsTrending" style="width:100%;"></div> -->
+                              <div id="TldsTrending" style="width:100%; height:800px;"></div>
+                          </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <div class="col-lg-12 tab-pane fade" id="domain-tab">
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <i class="fa fa-bar-chart-o fa-fw"></i> Top Domain Trending 
+                            <div class="pull-right">
+                                <div class="btn-group">
+                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
+                                        Actions
+                                        <span class="caret"></span>
+                                    </button>
+                                    <ul class="dropdown-menu pull-right" role="menu">
+                                        <li><a href="#" id="linear">Linear Scale</a>
+                                        </li>
+                                        <li><a href="#" id="log">Log Scale</a>
+                                        </li>
+                                        <li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
+                                        </li>
+                                        <li class="divider"></li>
+                                        <li><a href="#" id="edit_graph">Edit graph words</a>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div>
+
+                            <div class="btn-group btn-group-xs pull-right" style="margin-right: 5px;">
+                                <button type="button" class="btn btn-primary" onclick="take_top(5);">5</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(10);">10</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(15);">15</button>
+                            </div>
+
+                        </div>
+                        <!-- /.panel-heading -->
+                          <div class="panel-body">
+                             <!-- <div id="DomainTrending" style="width:100%;"></div> -->
+                              <div id="DomainTrending" style="width:100%; height:800px;"></div>
+                          </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+            </div> <!-- tab-content -->
+            <!-- /.row -->
+        </div>
+        <!-- /#page-wrapper -->
+
+        <!-- import graph function -->
+        <script src="{{ url_for('static', filename='js/plot-graph.js') }}"></script>
+
+        <!-- instanciate and plot graphs -->
+        <script type="text/javascript">
+            // Create, plot and set the limit of displayed headers
+            function create_and_plot(pannel, path){
+                //var graph_domain = new Graph($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
+                $.get(path, function(myContentFile) {
+                    var lines = myContentFile.split("\r\n");
+                    var header_size = lines[0].split(',').length-1;
+                    current_displayed_graph = new Graph(pannel, path, header_size);
+                    setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300);
+                }, 'text');
+                
+            }
+
+            // When a pannel is shown, create_and_plot.
+            $('.nav-tabs a').on('shown.bs.tab', function(event){
+                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));    
+            });
+        </script>
+
+        <script>
+            $(document).ready(function(){
+                // Create the graph when the page has just loaded
+                create_and_plot("TldsTrending", '../static//csv/tldstrendingdata.csv') 
+            });
+ 
+            // Used when we modify the number of displayed curves
+            function take_top(new_display){
+                current_displayed_graph.set_Visibility_andHide(new_display, default_display);
+                default_display = new_display;
+            }
+            
+        </script>
+
+    </div>
+    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</body>
+
+</html>

var/www/templates/Wordstrending.html

@@ -16,15 +16,19 @@
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-
+  <script> 
+      var default_display = {{ default_display }};
+      var current_displayed_graph;
+ </script>
 </head>
+
 <body>
 
     <div id="wrapper">
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
           <ul class="nav navbar-nav">
-             <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+             <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li></ul>
           </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
@@ -81,6 +85,13 @@
                                     </ul>
                                 </div>
                             </div>
+
+                            <div class="btn-group btn-group-xs pull-right" style="margin-right: 5px;">
+                                <button type="button" class="btn btn-primary" onclick="take_top(5);">5</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(10);">10</button>
+                                <button type="button" class="btn btn-primary" onclick="take_top(15);">15</button>
+                            </div>
+
                         </div>
                         <!-- /.panel-heading -->
                           <div class="panel-body">
@@ -93,101 +104,42 @@
             <!-- /.row -->
         </div>
         <!-- /#page-wrapper -->
-            <script type="text/javascript">
-g2 = new Dygraph(
-    document.getElementById("WordTrending"),
-    // path to CSV file
-    "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}",
-    //"../csv/wordstrendingdata.csv",
-    //window.csv,
-    {
-    rollPeriod: 1,
-    showRoller: true,
-    //drawPoints: true,
-    //fillGraph: true,
-    logscale: true,
-    animatedZooms: true,
-    labelsKMB: true,
-    highlightCircleSize: 3,
-    highlightSeriesOpts: {
-        strokeWidth: 3,
-        strokeBorderWidth: 1,
-        highlightCircleSize: 5,
-    },
-    underlayCallback: function(canvas, area, g) {
-    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
 
-    function highlight_period(x_start, x_end) {
-        var canvas_left_x = g.toDomXCoord(x_start);
-        var canvas_right_x = g.toDomXCoord(x_end);
-        var canvas_width = canvas_right_x - canvas_left_x;
-        canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
-    }
+        <!-- import graph function -->
+        <script src="{{ url_for('static', filename='js/plot-graph.js') }}"></script>
 
-    var min_data_x = g.getValue(0,0);
-    var max_data_x = g.getValue(g.numRows()-1,0);
+        <script type="text/javascript">
+            // Create, plot and set the limit of displayed headers
+            function create_and_plot(pannel, path){
+                //var graph_domain = new Graph($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
+                $.get(path, function(myContentFile) {
+                    var lines = myContentFile.split("\r\n");
+                    var header_size = lines[0].split(',').length-1;
+                    current_displayed_graph = new Graph(pannel, path, header_size);
+                    setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300);
+                }, 'text');
+                
+            }
 
-    // get day of week
-    var d = new Date(min_data_x);
-    var dow = d.getUTCDay();
-    var ds = d.toUTCString();
+            // When a pannel is shown, create_and_plot.
+            $('.nav-tabs a').on('shown.bs.tab', function(event){
+                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));    
+            });
+        </script>
 
-    var w = min_data_x;
-    // starting on Sunday is a special case
-    if (dow == 0) {
-        highlight_period(w,w+12*3600*1000);
-    }
-    // find first saturday
-    while (dow != 5) {
-        w += 24*3600*1000;
-        d = new Date(w);
-        dow = d.getUTCDay();
-    }
+        <script>
+            $(document).ready(function(){
+                // Create the graph when the page has just loaded
+                create_and_plot("WordTrending", '../static//csv/wordstrendingdata.csv') 
+            });
+ 
+            function take_top(new_display){
+                current_displayed_graph.set_Visibility_andHide(new_display, default_display);
+                default_display = new_display;
+            }
+            
+        </script>
 
-    // shift back 1/2 day to center highlight around the point for the day
-    w -= 12*3600*1000;
-    while (w < max_data_x) {
-        var start_x_highlight = w;
-        var end_x_highlight = w + 2*24*3600*1000;
-        // make sure we don't try to plot outside the graph
-        if (start_x_highlight < min_data_x) {
-            start_x_highlight = min_data_x;
-        }
-        if (end_x_highlight > max_data_x) {
-            end_x_highlight = max_data_x;
-        }
-        highlight_period(start_x_highlight,end_x_highlight);
-        // calculate start of highlight for next Saturday
-        w += 7*24*3600*1000;
-    }
-    }
-});
-
-onclick = function(ev) {
-    if (g2.isSeriesLocked()) {
-        g2.clearSelection();
-    }
-    else {
-        g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
-    }
-};
-g2.updateOptions({clickCallback: onclick}, true);
-
-var linear = document.getElementById("linear");
-var log = document.getElementById("log");
-linear.onclick = function() { setLog(false); }
-log.onclick = function() { setLog(true); }
-var setLog = function(val) {
-    g2.updateOptions({ logscale: val });
-    linear.disabled = !val;
-    log.disabled = val;
-    }
-function unzoomGraph() {
-    g2.updateOptions({
-        dateWindow:null,
-        valueRange:null
-    });
-}
 </script>
     </div>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>

var/www/templates/index.html

@@ -35,7 +35,7 @@
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
            <ul class="nav navbar-nav">
-           <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+           <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li></ul>
         </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">

var/www/templates/search.html

@@ -12,10 +12,31 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+  <!-- Custom style -->
+  <style> 
+      .tooltip-inner {
+         text-align: left;
+         height: 200%;
+         width: 200%;
+         max-width: 500px;
+         max-height: 500px;
+         font-size: 13px;
+      }
+      xmp {
+         white-space:pre-wrap;
+         word-wrap:break-word;
+      }
+      .modal-backdrop.fade {
+         opacity: 0;
+      }
+  </style>
 
 </head>
 <body>
@@ -24,7 +45,7 @@
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
           <ul class="nav navbar-nav">
-               <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+               <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li></ul>
           </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
@@ -39,24 +60,65 @@
           </div>
           <!-- /.navbar-static-side -->
       </nav>
+
+
+      <!-- Modal -->
+      <div id="mymodal" class="modal fade" role="dialog">
+        <div class="modal-dialog modal-lg">
+
+          <!-- Modal content-->
+          <div id="mymodalcontent" class="modal-content">
+            <div id="mymodalbody" class="modal-body" max-width="850px">
+              <p>Loading paste information...</p>
+              <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
+            </div>
+            <div class="modal-footer">
+              <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
+              <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+            </div>
+          </div>
+        </div>
+      </div>
+
+
         <div id="page-wrapper">
             <!-- /.row -->
             <div class="row"> </div>
             <div class="row">
                 <div class="col-lg-12">
+                    </br>
                     <div class="panel panel-default">
                         <div class="panel-heading">
-                            <i class="glyphicon glyphicon-search"></i> {{ r|length }} Results
+                            <i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
                             <div class="pull-right">
                             
                             </div>
                         </div>
                         <!-- /.panel-heading -->
                           <div class="panel-body">
-                          <table class="table">
-                          {% for result in r %}
-                                <tr><td>{{ result }}</td></tr>
+                          <table class="table table-striped table-bordered table-hover" id="myTable">
+                              <thead>
+                                   <tr>
+                                      <th>#</th>
+                                      <th style="max-width: 800px;">Path</th>
+                                      <th>Date</th>
+                                      <th>Size (Kb)</th>
+                                      <th>Action</th>
+                                   </tr>
+                              </thead>
+                              <tbody>
+                          {% set i = 0 %}
+                          {% for path in r %}
+                                <tr>
+                                    <td>{{ i + 1 }}</td>
+                                    <td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ path }}&num={{ i+1 }}"> {{ path }}</a></td>
+                                    <td>{{ paste_date[i] }}</td>
+                                    <td>{{ paste_size[i] }}</td>
+                                    <td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ c[i] }}"></span> <button type="button" class="btn-link" data-num="{{ i + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpaste') }}?paste={{ path }}&num={{ i+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
+                                </tr>
+                              {% set i = i + 1 %}
                           {% endfor %}
+                              </tbody>
                           </table>
                           </div>
                         <!-- /.panel-body -->
@@ -69,4 +131,89 @@
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
 </body>
 
+<!-- enable tooltip and dataTable -->
+<script>
+    $(document).ready(function(){
+        $('[data-toggle="tooltip"]').tooltip();
+        $("#button_show_path").hide();
+	$('#myTable').dataTable();
+    });
+</script>
+
+<!-- Dynamically update the modal -->
+<script type="text/javascript">
+    // static data
+    var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
+    var complete_paste = null;
+    var char_to_display = {{ char_to_display }};
+    var start_index = 0;
+
+    // On click, get html content from url and update the corresponding modal
+    $("[data-toggle='modal']").on("click", function (event) {
+        event.preventDefault();
+        var modal=$(this);
+        var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
+        $.get(url, function (data) {
+            $("#mymodalbody").html(data);
+            var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
+            button.tooltip();
+            $("#mymodalbody").children(".panel-default").append(button);
+
+            $("#button_show_path").attr('href', $(modal).attr('data-url'));
+            $("#button_show_path").show('fast');
+            $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+            if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                nothing_to_display();
+            }
+            // On click, donwload all paste's content
+            $("#load-more-button").on("click", function (event) {
+                if (complete_paste == null) { //Donwload only once
+                    $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                        complete_paste = data;
+                        update_preview();
+                    });
+                } else {
+                    update_preview();
+                }               
+            });
+        });
+    });
+
+    // When the modal goes out, refresh it to normal content
+    $("#mymodal").on('hidden.bs.modal', function () {
+        $("#mymodalbody").html("<p>Loading paste information...</p>");
+        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
+        $("#mymodalbody").append(loading_gif); // Show the loading GIF
+        $("#button_show_path").attr('href', '');
+        $("#button_show_path").hide();
+        complete_paste = null;
+        start_index = 0;
+    });
+    
+   // Update the paste preview in the modal
+   function update_preview() {
+       if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
+           var final_index = complete_paste.length-1;
+           var flag_stop = true;
+       } else {
+           var final_index = start_index + char_to_display;
+       }
+
+       if (final_index != start_index){ // still have data to display
+           $("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content
+           start_index = final_index; 
+           if (flag_stop)
+               nothing_to_display();
+       } else {
+          nothing_to_display(); 
+      }
+   } 
+   // Update the modal when there is no more data
+   function nothing_to_display() {
+       var new_content = $(alert_message).hide();
+       $("#mymodalbody").find("#panel-body").append(new_content);
+       new_content.show('fast');
+       $("#load-more-button").hide();
+   }
+</script>
 </html>

var/www/templates/searchbox.html

@@ -1,11 +1,16 @@
                           <div class="input-group custom-search-form">
-                              <form action="/search" method=POST>
+                              <form action="/search" id="form-search" method=POST>
                               <input type="text" name="query" class="form-control" placeholder="Search Paste">
                               <span class="input-group-btn">
                               <button class="btn btn-default" type="submit">
                                   <i class="fa fa-search"></i>
                               </button>
+                              <img id="loading-gif" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px; visibility: hidden;">
                               </form>
                           </span>
                           </div>
-
+                          <script> 
+                              $("#form-search").submit(function( event ) {
+                                  $("#loading-gif").css("visibility", "visible");
+                              });
+                          </script>

var/www/templates/show_saved_paste.html

@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Paste information</title>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+</head>
+<body>
+
+  <h2> Paste: {{ request.args.get('num') }}</h2>
+  <h3> {{ request.args.get('paste') }} </h3>
+
+  <hr></br>
+
+<div class="panel panel-default">
+  <div class="panel-heading">
+    <table class="table table-condensed">
+      <thead>
+        <tr>
+          <th>Date</th>
+          <th>Source</th>
+          <th>Encoding</th>
+          <th>Language</th>
+          <th>Size (Kb)</th>
+          <th>Mime</th>
+          <th>Number of lines</th>
+          <th>Max line length</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+        <td>{{ date }}</td>
+        <td>{{ source }}</td>
+        <td>{{ encoding }}</td>
+        <td>{{ language }}</td>
+        <td>{{ size }}</td>
+        <td>{{ mime }}</td>
+        <td>{{ lineinfo.0 }}</td>
+        <td>{{ lineinfo.1 }}</td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+  <div class="panel-body" id="panel-body">
+  <h4> Content: </h4>
+  <p data-initsize="{{ initsize }}"> <xmp id="paste-holder">{{ content }}</xmp></p>
+  </div>
+</div>
+
+
+</body>
+</html>

var/www/update_thirdparty.sh

@@ -24,6 +24,11 @@ wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
 
 wget http://www.goat1000.com/jquery.tagcanvas.js?2.5 -O ./static/js/jquery.tagcanvas.js
 
+#Ressources for dataTable
+wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
+wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
+wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
+
 rm -rf ./static/js/plugins
 mv temp/${filename}/js/* ./static/js/