Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -7,8 +7,6 @@ sudo: required
 
 dist: trusty
 
-group: edge
-
 addons:
     apt:
         packages:
@@ -27,6 +25,7 @@ addons:
             # Leveldb
             - libgmp-dev
             - libev-dev
+            - cmake
 
 env:
     - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
@@ -38,14 +37,14 @@ env:
 install:
     - pip install -U pip
     # DNS
-    - sudo apt-get install libadns1 libadns1-dev screen
+    - sudo apt-get install -y libadns1 libadns1-dev screen
     # required for mathplotlib
     - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
     - pip install distribute
     # Redis
     - test ! -d redis/ && git clone https://github.com/antirez/redis.git
     - pushd redis
-    - git checkout 3.0
+    - git checkout 3.2
     - make
     - popd
     # Redis leveldb
@@ -55,6 +54,20 @@ install:
     - git submodule update
     - make
     - popd
+    # Faup
+    - test ! -d faup && git clone https://github.com/stricaud/faup.git
+    - pushd faup/
+    - test ! -d build && mkdir build
+    - cd build
+    - cmake .. && make
+    - sudo make install
+    - echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
+    - sudo ldconfig
+    - popd
+    # PyFaup
+    - pushd faup/src/lib/bindings/python/
+    - python setup.py install
+    - popd
     # Set config
     - cp bin/packages/config.cfg.sample bin/packages/config.cfg
     - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}

bin/WebStats.py

@@ -6,22 +6,21 @@
 
 import time
 import datetime
-import re
 import redis
 import os
 from packages import lib_words
 from packages.Date import Date
 from pubsublogger import publisher
-from packages import Paste
 from Helper import Process
 from pyfaup.faup import Faup
 
 # Config Var
-threshold_need_to_look	= 50
+threshold_need_to_look = 50
-range_to_look		= 10
+range_to_look = 10
-threshold_to_plot	= 1 #500%
+threshold_to_plot = 1  # 500%
-to_plot			= set() 
+to_plot = set()
-clean_frequency		= 10 #minutes
+clean_frequency = 10  # minutes
+
 
 def analyse(server, field_name):
     field = url_parsed[field_name]
@@ -32,6 +31,7 @@ def analyse(server, field_name):
         else:
             server.hset(field, date, 1)
 
+
 def analyse_and_progression(server, field_name):
     field = url_parsed[field_name]
     if field is not None:
@@ -39,43 +39,34 @@ def analyse_and_progression(server, field_name):
         if prev_score is not None:
             print field + ' prev_score:' + prev_score
             server.hset(field, date, int(prev_score) + 1)
-            if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive
+            if int(prev_score) + 1 > threshold_need_to_look:  # threshold for false possitive
                 if(check_for_progression(server, field, date)):
                     to_plot.add(field)
         else:
             server.hset(field, date, 1)
 
+
 def check_for_progression(server, field, date):
     previous_data = set()
     tot_sum = 0
     for i in range(0, range_to_look):
         curr_value = server.hget(field, Date(date).substract_day(i))
-        if curr_value is None: #no further data
+        if curr_value is None:  # no further data
             break
         else:
             curr_value = int(curr_value)
             previous_data.add(curr_value)
-            tot_sum += curr_value 
+            tot_sum += curr_value
             if i == 0:
                 today_val = curr_value
 
-   
+    print 'totsum=' + str(tot_sum)
-    print 'totsum='+str(tot_sum)
+    print 'div=' + str(tot_sum / today_val)
-    print 'div='+str(tot_sum/today_val)
+    if tot_sum / today_val >= threshold_to_plot:
-    if tot_sum/today_val >= threshold_to_plot:
         return True
     else:
         return False
 
-def clean_to_plot():
-    temp_to_plot = set()
-    curr_date = datetime.date.today()
-    date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day))
-
-    for elem in to_plot:
-        if(check_for_progression(field, date)):
-            temp_to_plot.add(elem)    
-    to_plot = temp_to_plot
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@@ -98,7 +89,7 @@ if __name__ == '__main__':
         host=p.config.get("Redis_Level_DB", "host"),
         port=p.config.get("Redis_Level_DB", "port"),
         db=p.config.get("Redis_Level_DB", "db"))
- 
+
     r_serv2 = redis.StrictRedis(
         host=p.config.get("Redis_Level_DB_Domain", "host"),
         port=p.config.get("Redis_Level_DB_Domain", "port"),
@@ -106,18 +97,17 @@ if __name__ == '__main__':
 
     # FILE CURVE SECTION #
     csv_path_proto = os.path.join(os.environ['AIL_HOME'],
-                            p.config.get("Directories", "protocolstrending_csv"))
+                                  p.config.get("Directories", "protocolstrending_csv"))
     protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
-                                 p.config.get("Directories", "protocolsfile"))
+                                      p.config.get("Directories", "protocolsfile"))
-    
+
     csv_path_tld = os.path.join(os.environ['AIL_HOME'],
-                            p.config.get("Directories", "tldstrending_csv"))
+                                p.config.get("Directories", "tldstrending_csv"))
     tldsfile_path = os.path.join(os.environ['AIL_HOME'],
                                  p.config.get("Directories", "tldsfile"))
 
     csv_path_domain = os.path.join(os.environ['AIL_HOME'],
-                            p.config.get("Directories", "domainstrending_csv"))
+                                   p.config.get("Directories", "domainstrending_csv"))
-
 
     faup = Faup()
     generate_new_graph = False
@@ -125,7 +115,7 @@ if __name__ == '__main__':
     while True:
         # Get one message from the input queue
         message = p.get_from_set()
-        
+
         if message is None:
             if generate_new_graph:
                 generate_new_graph = False
@@ -143,8 +133,7 @@ if __name__ == '__main__':
                                                       month)
 
                 lib_words.create_curve_with_list(r_serv2, csv_path_domain,
-                                                      to_plot, year,
+                                                 to_plot, year, month)
-                                                      month)
                 print 'end building'
 
             publisher.debug("{} queue is empty, waiting".format(config_section))
@@ -152,13 +141,13 @@ if __name__ == '__main__':
             time.sleep(5)
             continue
 
-	else:
+        else:
             generate_new_graph = True
             # Do something with the message from the queue
             url, date = message.split()
             faup.decode(url)
             url_parsed = faup.get()
-            
+
-            analyse(r_serv1, 'scheme')	#Scheme analysis
+            analyse(r_serv1, 'scheme')  # Scheme analysis
-            analyse(r_serv1, 'tld')	#Tld analysis
+            analyse(r_serv1, 'tld')     # Tld analysis
-	    analyse_and_progression(r_serv2, 'domain')	#Domain analysis
+            analyse_and_progression(r_serv2, 'domain')  # Domain analysis

bin/launch_scripts.sh

@@ -35,4 +35,5 @@ sleep 0.1
 screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
 sleep 0.1
 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
-
+sleep 0.1
+screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'

bin/packages/config.cfg.sample

@@ -1,9 +1,30 @@
 [Directories]
 bloomfilters = Blooms
 pastes = PASTES
+
 wordtrending_csv = var/www/static/csv/wordstrendingdata
 wordsfile = files/wordfile
 
+protocolstrending_csv = var/www/static/csv/protocolstrendingdata
+protocolsfile = files/protocolsfile
+
+tldstrending_csv = var/www/static/csv/tldstrendingdata
+tldsfile = AILENV/faup/src/data/mozilla.tlds
+
+domainstrending_csv = var/www/static/csv/domainstrendingdata
+
+##### Flask #####
+[Flask]
+#Number of minutes displayed for the number of processed pastes.
+minute_processed_paste = 10
+
+#Maximum number of character to display in the toolip
+max_preview_char = 250
+#Maximum number of character to display in the modal
+max_preview_modal = 800
+#Default number of header to display in trending graphs
+default_display = 10
+
 ##### Redis #####
 [Redis_Cache]
 host = localhost
@@ -28,12 +49,17 @@ db = 1
 ##### LevelDB #####
 [Redis_Level_DB]
 host = localhost
-port = 2013
+port = 2016
 db = 0
 
+[Redis_Level_DB_Domain]
+host = localhost
+port = 2016
+db = 3
+
 [Redis_Level_DB_Hashs]
 host = localhost
-port = 2013
+port = 2016
 db = 1
 
 [Url]

installing_deps.sh

@@ -6,7 +6,7 @@ set -x
 sudo apt-get update
 
 sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
-    screen g++ python-tk unzip libsnappy-dev
+    screen g++ python-tk unzip libsnappy-dev cmake
 
 #Needed for bloom filters
 sudo apt-get install libssl-dev libfreetype6-dev python-numpy
@@ -24,10 +24,21 @@ sudo easy_install -U distribute
 # REDIS #
 test ! -d redis/ && git clone https://github.com/antirez/redis.git
 pushd redis/
-git checkout 3.0
+git checkout 3.2
 make
 popd
 
+# Faup
+test ! -d faup && git clone https://github.com/stricaud/faup.git
+pushd faup/
+test ! -d build && mkdir build
+cd build
+cmake .. && make
+sudo make install
+echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
+sudo ldconfig
+popd
+
 # REDIS LEVEL DB #
 test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
 pushd redis-leveldb/
@@ -56,5 +67,11 @@ mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
 pip install -U pip
 pip install -r pip_packages_requirement.txt
 
+# Pyfaup
+pushd faup/src/lib/bindings/python/
+python setup.py install
+popd
+
+
 # Download the necessary NLTK corpora
 HOME=$(pwd) python -m textblob.download_corpora

var/www/static/js/indexjavascript.js

@@ -1,52 +1,11 @@
 // Plot and update the number of processed pastes
 $(function() {
     var data = [];
-    var totalPoints = 60*10; //60s*10m
+    var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
+    var totalPoints = 60*parseInt(default_minute); //60s*minute
     var curr_max = 0;
     
     function getData() {
-        if (data.length > 0){
-             var data_old = data[0];
-             data = data.slice(1);
-             curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
-        }
-        
-        while (data.length < totalPoints) {
-            var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
-            curr_max = y > curr_max ? y : curr_max;
-            data.push(y);
-        }
-        // Zip the generated y values with the x values
-        var res = [];
-        for (var i = 0; i < data.length; ++i) {
-            res.push([i, data[i]])
-        }
-        return res;
-    }
-
-    var updateInterval = 1000;
-    var options = {
-        series: { shadowSize: 1 },
-        lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
-        yaxis: { min: 0, max: 40 },
-        colors: ["#a971ff"],
-        grid: {
-            tickColor: "#dddddd",
-            borderWidth: 0 
-        },
-    };
-    var plot = $.plot("#realtimechart", [ getData() ], options);
-    
-    function update() {
-        plot.setData([getData()]);
-        plot.getOptions().yaxes[0].max = curr_max;
-        plot.setupGrid();
-        plot.draw();
-        setTimeout(update, updateInterval);
-    }
-    update();
-});
-
 function initfunc( csvay, scroot) {
   window.csv = csvay;
   window.scroot = scroot;

var/www/templates/Queue_live_Monitoring.html

@@ -1,7 +1,6 @@
 <html>
 <head>
 <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
 </head>
 <body>
 <title>WordsTrend</title>

var/www/templates/Trending.html

@@ -14,9 +14,8 @@
 	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
 	  <!-- JS -->
 	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-	  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
 	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-          <script> 
+          <script>
               var default_display = {{ default_display }};
               var current_displayed_graph;
           </script>
@@ -56,7 +55,7 @@
           </div>
             <!-- /.row -->
             <div class="row">
-            
+
             <!-- /.nav-tabs -->
             <ul class="nav nav-tabs">
                 <li class="active"><a data-toggle="tab" href="#tld-tab" data-pannel="TldTrending" data-path="../static//csv/tldstrendingdata.csv">Top level domains</a></li>
@@ -65,8 +64,8 @@
                 <li><a data-toggle="tab" href="#protocol-tab" data-pannel="ProtocolTrending" data-path="../static//csv/protocolstrendingdata.csv">Protocols</a></li>
             </ul>
             </br>
-              
+
-            <div class="tab-content">  
+            <div class="tab-content">
                 <div class="col-lg-12 tab-pane fade in active" id="tld-tab" >
                      {% include 'trending_graphs/Tldstrending.html' %}
                 </div>
@@ -75,7 +74,7 @@
                 </div>
                 <div class="col-lg-12 tab-pane fade" id="words-tab">
                      {% include 'trending_graphs/Wordstrending.html' %}
-                </div> 
+                </div>
                 <div class="col-lg-12 tab-pane fade" id="protocol-tab">
                      {% include 'trending_graphs/Protocolstrending.html' %}
                 </div>
@@ -98,27 +97,27 @@
                     current_displayed_graph = new Graph(pannel, path, header_size);
                     setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300);
                 }, 'text');
-                
+
             }
 
             // When a pannel is shown, create_and_plot.
             $('.nav-tabs a').on('shown.bs.tab', function(event){
-                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));    
+                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
             });
         </script>
 
         <script>
             $(document).ready(function(){
                 // Create the graph when the page has just loaded
-                create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv') 
+                create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv')
             });
- 
+
             // Used when we modify the number of displayed curves
             function take_top(new_display){
                 current_displayed_graph.set_Visibility_andHide(new_display, default_display);
                 default_display = new_display;
             }
-            
+
         </script>
 
     </div>

var/www/templates/index.html

@@ -14,11 +14,10 @@
   <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.tagcanvas.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
   <script>
+      window.default_minute = {{ default_minute }};
       function update_values() {
           $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
           $.getJSON($SCRIPT_ROOT+"/_stuff",
@@ -108,7 +107,7 @@
 		<input id="checkbox_log_info" type="checkbox" value="info" checked="true"> INFO
 		<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
 		<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL
- 
+
             </div>
         </div>
         <div class="panel-body">

var/www/templates/search.html

@@ -15,12 +15,11 @@
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
   <!-- Custom style -->
-  <style> 
+  <style>
       .tooltip-inner {
          text-align: left;
          height: 200%;
@@ -93,7 +92,7 @@
                         <div class="panel-heading">
                             <i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
                             <div class="pull-right">
-                            
+
                             </div>
                         </div>
                         <!-- /.panel-heading -->
@@ -176,7 +175,7 @@
                     });
                 } else {
                     update_preview();
-                }               
+                }
             });
         });
     });
@@ -191,7 +190,7 @@
         complete_paste = null;
         start_index = 0;
     });
-    
+
    // Update the paste preview in the modal
    function update_preview() {
        if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
@@ -203,13 +202,13 @@
 
        if (final_index != start_index){ // still have data to display
            $("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content
-           start_index = final_index; 
+           start_index = final_index;
            if (flag_stop)
                nothing_to_display();
        } else {
-          nothing_to_display(); 
+          nothing_to_display();
       }
-   } 
+   }
    // Update the modal when there is no more data
    function nothing_to_display() {
        var new_content = $(alert_message).hide();

var/www/update_thirdparty.sh

@@ -4,26 +4,30 @@ set -e
 
 wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
 
-SBADMIN_VERSION=2
+SBADMIN_VERSION='1.0.4'
-filename="sb-admin-${SBADMIN_VERSION}"
 
 rm -rf temp
 mkdir temp
 
-wget https://github.com/IronSummitMedia/startbootstrap-sb-admin-2/archive/v1.0.2.zip -O temp/${filename}".zip"
+wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
-unzip temp/${filename}".zip" -d temp/
+unzip temp/${SBADMIN_VERSION}.zip -d temp/
-mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
+mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin-2
 
-JQVERSION="1.11.1"
+rm -rf ./static/js/plugins
+mv temp/sb-admin-2/js/* ./static/js/
+
+rm -rf ./static/fonts/ ./static/font-awesome/
+mv temp/sb-admin-2/fonts/ ./static/
+mv temp/sb-admin-2/font-awesome/ ./static/
+
+rm -rf ./static/css/plugins/
+mv temp/sb-admin-2/css/* ./static/css/
+
+rm -rf temp
+
+JQVERSION="1.12.4"
 wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
 
-#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
-
-#Here to fix an error about an hard dependency in a obscur script of bootstrap..
-wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
-
-wget http://www.goat1000.com/jquery.tagcanvas.js?2.5 -O ./static/js/jquery.tagcanvas.js
-
 #Ressources for dataTable
 wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
 wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
@@ -33,21 +37,7 @@ wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTabl
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
 
-rm -rf ./static/js/plugins
-mv temp/${filename}/js/* ./static/js/
-
-rm -rf ./static/fonts/ ./static/font-awesome-4.1.0/
-
-mv temp/${filename}/fonts/ ./static/
-mv temp/${filename}/font-awesome/ ./static/
-
-rm -rf ./static/css/plugins/
-mv temp/${filename}/css/* ./static/css/
-
-rm -rf temp/
 mkdir -p ./static/image
-cd static/image
+pushd static/image
 wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
-
+popd
-cd ../..
-