Merge pull request #56 from mokaddem/mokaddem-testing

New statistics modules and usage of Faup

bin/Categ.py

@@ -66,7 +66,7 @@ if __name__ == "__main__":
     # FUNCTIONS #
     publisher.info("Script Categ started")
 
-    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
+    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
     tmp_dict = {}
     for filename in categories:
         bname = os.path.basename(filename)

bin/Cve.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+    Template for new modules
+"""
+
+import time
+import re
+from pubsublogger import publisher
+from packages import Paste
+from Helper import Process
+
+
+def search_cve(message):
+    filepath, count = message.split()
+    paste = Paste.Paste(filepath)
+    content = paste.get_p_content()
+    # regex to find CVE
+    reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
+    # list of the regex results in the Paste, may be null
+    results = set(reg_cve.findall(content))
+
+    # if the list is greater than 2, we consider the Paste may contain a list of cve
+    if len(results) > 0:
+        print('{} contains CVEs'.format(paste.p_name))
+        publisher.warning('{} contains CVEs'.format(paste.p_name))
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'Cve'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("Run CVE module")
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            time.sleep(1)
+            continue
+
+        # Do something with the message from the queue
+        search_cve(message)
+
+        # (Optional) Send that thing to the next queue
+        #p.populate_set_out(something_has_been_done)

bin/DomClassifier.py

@@ -39,7 +39,7 @@ def main():
             if message is not None:
                 PST = Paste.Paste(message)
             else:
-                publisher.debug("Script DomClassifier is idling 10s")
+                publisher.debug("Script DomClassifier is idling 1s")
                 time.sleep(1)
                 continue
             paste = PST.get_p_content()

bin/Duplicate.py

@@ -33,11 +33,12 @@ if __name__ == "__main__":
     # DB OBJECT & HASHS ( DISK )
     # FIXME increase flexibility
     dico_redis = {}
-    for year in xrange(2013, 2015):
+    for year in xrange(2013, 2017):
         for month in xrange(0, 16):
             dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
                 host=p.config.get("Redis_Level_DB", "host"), port=year,
                 db=month)
+	    #print("dup: "+str(year)+str(month).zfill(2)+"\n")
 
     # FUNCTIONS #
     publisher.info("Script duplicate started")
@@ -81,7 +82,7 @@ if __name__ == "__main__":
                 bloop_path_set.add(filebloompath)
 
             # UNIQUE INDEX HASHS TABLE
-            r_serv0 = dico_redis["201300"]
+            r_serv0 = dico_redis["201600"]
             r_serv0.incr("current_index")
             index = r_serv0.get("current_index")+str(PST.p_date)
             # HASHTABLES PER MONTH (because of r_serv1 changing db)

bin/LAUNCH.sh

@@ -66,6 +66,7 @@ function launching_lvldb {
     lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
     db1_y='2013'
     db2_y='2014'
+    db3_y='2016'
     nb_db=13
 
     screen -dmS "LevelDB"
@@ -75,6 +76,8 @@ function launching_lvldb {
     screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
     screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {
@@ -127,6 +130,16 @@ function launching_scripts {
     screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
 }
 
 #If no params, display the help

bin/Url.py

@@ -7,6 +7,8 @@ import dns.exception
 from packages import Paste
 from packages import lib_refine
 from pubsublogger import publisher
+from pyfaup.faup import Faup
+import re
 
 # Country and ASN lookup
 from cymru.ip2asn.dns import DNSClient as ip2asn
@@ -16,6 +18,13 @@ import ipaddress
 
 from Helper import Process
 
+# Used to prevent concat with empty fields due to url parsing
+def avoidNone(str):
+    if str is None:
+        return ""
+    else:
+        return str
+
 if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"
@@ -41,6 +50,7 @@ if __name__ == "__main__":
 
     message = p.get_from_set()
     prec_filename = None
+    faup = Faup()
 
     url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
 
@@ -53,19 +63,27 @@ if __name__ == "__main__":
                 PST = Paste.Paste(filename)
                 client = ip2asn()
                 for x in PST.get_regex(url_regex):
-                    scheme, credential, subdomain, domain, host, tld, \
+                    matching_url = re.search(url_regex, PST.get_p_content())
-                        port, resource_path, query_string, f1, f2, f3, \
+                    url = matching_url.group(0)
-                        f4 = x
+
+                    to_send = "{} {}".format(url, PST._get_p_date())
+                    p.populate_set_out(to_send, 'Url')
+
+                    faup.decode(url)
+                    domain = faup.get_domain()
+                    subdomain = faup.get_subdomain()
+                    f1 = None
+
                     domains_list.append(domain)
-                    p.populate_set_out(x, 'Url')
+
-                    publisher.debug('{} Published'.format(x))
+                    publisher.debug('{} Published'.format(url))
 
                     if f1 == "onion":
                         print domain
 
-                    hostl = unicode(subdomain+domain)
+                    hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
                     try:
-                        socket.setdefaulttimeout(2)
+                        socket.setdefaulttimeout(1)
                         ip = socket.gethostbyname(unicode(hostl))
                     except:
                         # If the resolver is not giving any IPv4 address,

bin/WebStats.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+    Template for new modules
+"""
+
+import time
+import datetime
+import re
+import redis
+import os
+from packages import lib_words
+from pubsublogger import publisher
+from packages import Paste
+from Helper import Process
+from pyfaup.faup import Faup
+
+def analyse(field_name):
+    field = url_parsed[field_name]
+    if field is not None:
+        prev_score = r_serv1.hget(field, date)
+        if prev_score is not None:
+            r_serv1.hset(field, date, int(prev_score) + 1)
+        else:
+            r_serv1.hset(field, date, 1)
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'WebStats'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("Makes statistics about valid URL")
+
+    # REDIS #
+    r_serv1 = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB", "host"),
+        port=p.config.get("Redis_Level_DB", "port"),
+        db=p.config.get("Redis_Level_DB", "db"))
+
+    # FILE CURVE SECTION #
+    csv_path_proto = os.path.join(os.environ['AIL_HOME'],
+                            p.config.get("Directories", "protocolstrending_csv"))
+    protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
+                                 p.config.get("Directories", "protocolsfile"))
+    
+    csv_path_tld = os.path.join(os.environ['AIL_HOME'],
+                            p.config.get("Directories", "tldstrending_csv"))
+    tldsfile_path = os.path.join(os.environ['AIL_HOME'],
+                                 p.config.get("Directories", "tldsfile"))
+
+    faup = Faup()
+    generate_new_graph = False
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+        
+        if message is None:
+            if generate_new_graph:
+                generate_new_graph = False
+                print 'Building graph'
+                today = datetime.date.today()
+                year = today.year
+                month = today.month
+                
+                lib_words.create_curve_with_word_file(r_serv1, csv_path_proto,
+                                                      protocolsfile_path, year,
+                                                      month)
+
+                lib_words.create_curve_with_word_file(r_serv1, csv_path_tld,
+                                                      tldsfile_path, year,
+                                                      month)
+
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            time.sleep(1)
+            continue
+
+	else:
+            generate_new_graph = True
+            # Do something with the message from the queue
+            url, date = message.split()
+            faup.decode(url)
+            url_parsed = faup.get()
+            
+            analyse('scheme')	#Scheme analysis
+            analyse('tld')	#Tld analysis

bin/empty_queue.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+"""
+The Empty queue module
+====================
+
+This simple module can be used to clean all queues.
+
+Requirements:
+-------------
+
+
+"""
+import redis
+import os
+import time
+from packages import Paste
+from pubsublogger import publisher
+from Helper import Process
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web',  'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
+
+    for queue in config_section:
+        print 'dropping: ' + queue
+        p = Process(queue)
+        while True:
+            message = p.get_from_set()
+            if message is None:
+                break
+

bin/packages/modules.cfg

@@ -27,7 +27,7 @@ subscribe = Redis_Words
 
 [Categ]
 subscribe = Redis_Global
-publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
+publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
 
 [CreditCards]
 subscribe = Redis_CreditCards
@@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
 subscribe = Redis_Web
 publish = Redis_Url,ZMQ_Url
 
+[WebStats]
+subscribe = Redis_Url
+
 [Release]
 subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
 
+[Cve]
+subscribe = Redis_Cve
+
 [Phone]
 subscribe = Redis_Global
 

files/Cve

@@ -0,0 +1 @@
+CVE

files/protocolsfile

@@ -0,0 +1,100 @@
+afs
+file
+ftp
+z39.50
+z39.50r
+z39.50s
+vemmi
+urn
+nfs
+dict
+acap
+rtspu
+rtsp
+rtsps
+tip
+pop
+cid
+mid
+data
+thismessage
+service
+shttp
+fax
+modem
+tv
+sip
+sips
+go
+icap
+h323
+ipp
+xmlrpc.beep
+xmlrpc.beeps
+tftp
+mupdate
+pres
+im
+mtqp
+tel
+iris
+iris.beep
+crid
+snmp
+tag
+wais
+prospero
+soap.beep
+soap.beeps
+telnet
+gopher
+cap
+info
+dns
+ldap
+dav
+opaquelocktoken
+msrp
+msrps
+dtn
+imap
+xmpp
+iax
+news
+nntp
+snews
+sms
+rsync
+sieve
+geo
+mailto
+jms
+mailserver
+ipn
+tn3270
+ws
+wss
+xcon
+xcon-userid
+about
+aaa
+aaas
+session
+ni
+nih
+reload
+ham
+stun
+stuns
+turn
+turns
+http
+https
+coap
+coaps
+rtmfp
+ipps
+pkcs11
+acct
+example	
+vnc

files/tldsfile

@@ -0,0 +1,263 @@
+com
+org
+net
+int
+edu
+gov
+mil
+arpa
+ac
+ad
+ae
+af
+ag
+ai
+al
+am
+an
+ao
+aq
+ar
+as
+at
+au
+aw
+ax
+az
+ba
+bb
+bd
+be
+bf
+bg
+bh
+bi
+bj
+bl
+bm
+bn
+bo
+bq
+br
+bs
+bt
+bv
+bw
+by
+bz
+ca
+cc
+cd
+cf
+cg
+ch
+ci
+ck
+cl
+cm
+cn
+co
+cr
+cu
+cv
+cw
+cx
+cy
+cz
+de
+dj
+dk
+dm
+do
+dz
+ec
+ee
+eg
+eh
+er
+es
+et
+eu
+fi
+fj
+fk
+fm
+fo
+fr
+ga
+gb
+gd
+ge
+gf
+gg
+gh
+gi
+gl
+gm
+gn
+gp
+gq
+gr
+gs
+gt
+gu
+gw
+gy
+hk
+hm
+hn
+hr
+ht
+hu
+id
+ie
+il
+im
+in
+io
+iq
+ir
+is
+it
+je
+jm
+jo
+jp
+ke
+kg
+kh
+ki
+km
+kn
+kp
+kr
+kw
+ky
+kz
+la
+lb
+lc
+li
+lk
+lr
+ls
+lt
+lu
+lv
+ly
+ma
+mc
+md
+me
+mf
+mg
+mh
+mk
+ml
+mm
+mn
+mo
+mp
+mq
+mr
+ms
+mt
+mu
+mv
+mw
+mx
+my
+mz
+na
+nc
+ne
+nf
+ng
+ni
+nl
+no
+np
+nr
+nu
+nz
+om
+pa
+pe
+pf
+pg
+ph
+pk
+pl
+pm
+pn
+pr
+ps
+pt
+pw
+py
+qa
+re
+ro
+rs
+ru
+rw
+sa
+sb
+sc
+sd
+se
+sg
+sh
+si
+sj
+sk
+sl
+sm
+sn
+so
+sr
+ss
+st
+su
+sv
+sx
+sy
+sz
+tc
+td
+tf
+tg
+th
+tj
+tk
+tl
+tm
+tn
+to
+tp
+tr
+tt
+tv
+tw
+tz
+ua
+ug
+uk
+um
+us
+uy
+uz
+va
+vc
+ve
+vg
+vi
+vn
+vu
+wf
+ws
+ye
+yt
+za
+zm
+zw

var/www/Flask_server.py

@@ -96,5 +96,14 @@ def wordstrending():
     return render_template("Wordstrending.html")
 
 
+@app.route("/protocolstrending/")
+def protocolstrending():
+    return render_template("Protocolstrending.html")
+
+@app.route("/tldstrending/")
+def tldstrending():
+    return render_template("Tldstrending.html")
+
+
 if __name__ == "__main__":
     app.run(host='0.0.0.0', port=7000, threaded=True)

var/www/templates/Protocolstrending.html

@@ -0,0 +1,196 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+  <!-- JS -->
+  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+
+</head>
+<body>
+
+    <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+          <div class="navbar-header">
+          <ul class="nav navbar-nav">
+            <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+         </div>
+          <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          <div class="input-group custom-search-form">
+                              <input type="text" class="form-control" placeholder="Search Paste">
+                              <span class="input-group-btn">
+                              <button class="btn btn-default" type="button">
+                                  <i class="fa fa-search"></i>
+                              </button>
+                          </span>
+                          </div>
+                          <!-- /input-group -->
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+        <div id="page-wrapper">
+          <div class="row">
+              <div class="col-lg-12">
+                  <h1 class="page-header">ProtocolsTrendings</h1>
+              </div>
+              <!-- /.col-lg-12 -->
+          </div>
+            <!-- /.row -->
+            <div class="row">
+                <div class="col-lg-12">
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <i class="fa fa-bar-chart-o fa-fw"></i> Protocols Trend
+                            <div class="pull-right">
+                                <div class="btn-group">
+                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
+                                        Actions
+                                        <span class="caret"></span>
+                                    </button>
+                                    <ul class="dropdown-menu pull-right" role="menu">
+                                        <li><a href="#" id="linear">Linear Scale</a>
+                                        </li>
+                                        <li><a href="#" id="log">Log Scale</a>
+                                        </li>
+                                        <li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
+                                        </li>
+                                        <li class="divider"></li>
+                                        <li><a href="#" id="edit_graph">Edit graph words</a>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div>
+                        </div>
+                        <!-- /.panel-heading -->
+                          <div class="panel-body">
+                             <!-- <div id="ProtocolsTrending" style="width:100%;"></div> -->
+                              <div id="ProtocolsTrending" style="width:100%; height:800px;"></div>
+                          </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+            <!-- /.row -->
+        </div>
+        <!-- /#page-wrapper -->
+            <script type="text/javascript">
+g2 = new Dygraph(
+    document.getElementById("ProtocolsTrending"),
+    // path to CSV file
+    "{{ url_for('static', filename='csv/protocolstrendingdata.csv') }}",
+    //"../csv/wordstrendingdata.csv",
+    //window.csv,
+    {
+    rollPeriod: 1,
+    showRoller: true,
+    //drawPoints: true,
+    //fillGraph: true,
+    logscale: true,
+    animatedZooms: true,
+    labelsKMB: true,
+    highlightCircleSize: 3,
+    highlightSeriesOpts: {
+        strokeWidth: 3,
+        strokeBorderWidth: 1,
+        highlightCircleSize: 5,
+    },
+    underlayCallback: function(canvas, area, g) {
+    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
+
+    function highlight_period(x_start, x_end) {
+        var canvas_left_x = g.toDomXCoord(x_start);
+        var canvas_right_x = g.toDomXCoord(x_end);
+        var canvas_width = canvas_right_x - canvas_left_x;
+        canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
+    }
+
+    var min_data_x = g.getValue(0,0);
+    var max_data_x = g.getValue(g.numRows()-1,0);
+
+    // get day of week
+    var d = new Date(min_data_x);
+    var dow = d.getUTCDay();
+    var ds = d.toUTCString();
+
+    var w = min_data_x;
+    // starting on Sunday is a special case
+    if (dow == 0) {
+        highlight_period(w,w+12*3600*1000);
+    }
+    // find first saturday
+    while (dow != 5) {
+        w += 24*3600*1000;
+        d = new Date(w);
+        dow = d.getUTCDay();
+    }
+
+    // shift back 1/2 day to center highlight around the point for the day
+    w -= 12*3600*1000;
+    while (w < max_data_x) {
+        var start_x_highlight = w;
+        var end_x_highlight = w + 2*24*3600*1000;
+        // make sure we don't try to plot outside the graph
+        if (start_x_highlight < min_data_x) {
+            start_x_highlight = min_data_x;
+        }
+        if (end_x_highlight > max_data_x) {
+            end_x_highlight = max_data_x;
+        }
+        highlight_period(start_x_highlight,end_x_highlight);
+        // calculate start of highlight for next Saturday
+        w += 7*24*3600*1000;
+    }
+    }
+});
+
+onclick = function(ev) {
+    if (g2.isSeriesLocked()) {
+        g2.clearSelection();
+    }
+    else {
+        g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
+    }
+};
+g2.updateOptions({clickCallback: onclick}, true);
+
+var linear = document.getElementById("linear");
+var log = document.getElementById("log");
+linear.onclick = function() { setLog(false); }
+log.onclick = function() { setLog(true); }
+var setLog = function(val) {
+    g2.updateOptions({ logscale: val });
+    linear.disabled = !val;
+    log.disabled = val;
+    }
+function unzoomGraph() {
+    g2.updateOptions({
+        dateWindow:null,
+        valueRange:null
+    });
+}
+</script>
+    </div>
+    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</body>
+
+</html>

var/www/templates/Tldstrending.html

@@ -0,0 +1,196 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+  <!-- JS -->
+  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+
+</head>
+<body>
+
+    <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+          <div class="navbar-header">
+          <ul class="nav navbar-nav">
+              <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
+          </div>
+          <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          <div class="input-group custom-search-form">
+                              <input type="text" class="form-control" placeholder="Search Paste">
+                              <span class="input-group-btn">
+                              <button class="btn btn-default" type="button">
+                                  <i class="fa fa-search"></i>
+                              </button>
+                          </span>
+                          </div>
+                          <!-- /input-group -->
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+        <div id="page-wrapper">
+          <div class="row">
+              <div class="col-lg-12">
+                  <h1 class="page-header">Top Level Domain Trending</h1>
+              </div>
+              <!-- /.col-lg-12 -->
+          </div>
+            <!-- /.row -->
+            <div class="row">
+                <div class="col-lg-12">
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
+                            <div class="pull-right">
+                                <div class="btn-group">
+                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
+                                        Actions
+                                        <span class="caret"></span>
+                                    </button>
+                                    <ul class="dropdown-menu pull-right" role="menu">
+                                        <li><a href="#" id="linear">Linear Scale</a>
+                                        </li>
+                                        <li><a href="#" id="log">Log Scale</a>
+                                        </li>
+                                        <li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
+                                        </li>
+                                        <li class="divider"></li>
+                                        <li><a href="#" id="edit_graph">Edit graph words</a>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div>
+                        </div>
+                        <!-- /.panel-heading -->
+                          <div class="panel-body">
+                             <!-- <div id="WordTrending" style="width:100%;"></div> -->
+                              <div id="TldsTrending" style="width:100%; height:800px;"></div>
+                          </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+            <!-- /.row -->
+        </div>
+        <!-- /#page-wrapper -->
+            <script type="text/javascript">
+g2 = new Dygraph(
+    document.getElementById("TldsTrending"),
+    // path to CSV file
+    "{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
+    //"../csv/wordstrendingdata.csv",
+    //window.csv,
+    {
+    rollPeriod: 1,
+    showRoller: true,
+    //drawPoints: true,
+    //fillGraph: true,
+    logscale: true,
+    animatedZooms: true,
+    labelsKMB: true,
+    highlightCircleSize: 3,
+    highlightSeriesOpts: {
+        strokeWidth: 3,
+        strokeBorderWidth: 1,
+        highlightCircleSize: 5,
+    },
+    underlayCallback: function(canvas, area, g) {
+    canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
+
+    function highlight_period(x_start, x_end) {
+        var canvas_left_x = g.toDomXCoord(x_start);
+        var canvas_right_x = g.toDomXCoord(x_end);
+        var canvas_width = canvas_right_x - canvas_left_x;
+        canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
+    }
+
+    var min_data_x = g.getValue(0,0);
+    var max_data_x = g.getValue(g.numRows()-1,0);
+
+    // get day of week
+    var d = new Date(min_data_x);
+    var dow = d.getUTCDay();
+    var ds = d.toUTCString();
+
+    var w = min_data_x;
+    // starting on Sunday is a special case
+    if (dow == 0) {
+        highlight_period(w,w+12*3600*1000);
+    }
+    // find first saturday
+    while (dow != 5) {
+        w += 24*3600*1000;
+        d = new Date(w);
+        dow = d.getUTCDay();
+    }
+
+    // shift back 1/2 day to center highlight around the point for the day
+    w -= 12*3600*1000;
+    while (w < max_data_x) {
+        var start_x_highlight = w;
+        var end_x_highlight = w + 2*24*3600*1000;
+        // make sure we don't try to plot outside the graph
+        if (start_x_highlight < min_data_x) {
+            start_x_highlight = min_data_x;
+        }
+        if (end_x_highlight > max_data_x) {
+            end_x_highlight = max_data_x;
+        }
+        highlight_period(start_x_highlight,end_x_highlight);
+        // calculate start of highlight for next Saturday
+        w += 7*24*3600*1000;
+    }
+    }
+});
+
+onclick = function(ev) {
+    if (g2.isSeriesLocked()) {
+        g2.clearSelection();
+    }
+    else {
+        g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
+    }
+};
+g2.updateOptions({clickCallback: onclick}, true);
+
+var linear = document.getElementById("linear");
+var log = document.getElementById("log");
+linear.onclick = function() { setLog(false); }
+log.onclick = function() { setLog(true); }
+var setLog = function(val) {
+    g2.updateOptions({ logscale: val });
+    linear.disabled = !val;
+    log.disabled = val;
+    }
+function unzoomGraph() {
+    g2.updateOptions({
+        dateWindow:null,
+        valueRange:null
+    });
+}
+</script>
+    </div>
+    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</body>
+
+</html>

var/www/templates/Wordstrending.html

@@ -24,7 +24,7 @@
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
           <ul class="nav navbar-nav">
-           <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li class="active"><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
+             <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
           </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">

var/www/templates/index.html

@@ -35,7 +35,7 @@
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
            <ul class="nav navbar-nav">
-           <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
+           <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
         </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">

var/www/templates/search.html

@@ -24,7 +24,7 @@
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
           <div class="navbar-header">
           <ul class="nav navbar-nav">
-           <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
+               <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
           </div>
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">

var/www/update_thirdparty.sh

@@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
 JQVERSION="1.11.1"
 wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
 
-wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
+#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
 
 #Here to fix an error about an hard dependency in a obscur script of bootstrap..
 wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js