Merge branch 'bugfix'

.gitignore

@@ -20,6 +20,7 @@ var/www/static/
 !var/www/static/js/moduleTrending.js
 !var/www/static/js/plot-graph.js
 !var/www/static/js/trendingchart.js
+var/www/templates/header.html # auto-generated
 
 # Local config
 bin/packages/config.cfg
@@ -27,3 +28,4 @@ bin/packages/config.cfg
 # installed files
 nltk_data/
 doc/all_modules.txt
+doc/module-data-flow.png # auto-generated

bin/LAUNCH.sh

@@ -66,32 +66,36 @@ function launching_redis {
 
 function launching_lvldb {
     #Want to launch more level_db?
+    #FIXME update the date in config.cfg
     lvdbhost='127.0.0.1'
     lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
-    db1_y='2013'
-    db2_y='2014'
-    db3_y='2016'
-    db4_y='2017'
+    db1_y='2016'
+    db2_y='2017'
+    dbn_y=`date +%Y`
 
-    dbC_y='3016'
+    dbC1_y='3016'
+    dbCn_y=30`date +%y`
     nb_db=13
 
     screen -dmS "LevelDB"
     sleep 0.1
     echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
     #Add lines here with appropriates options.
-    screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db2_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "$dbn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbn_y'/ -P '$dbn_y' -M '$nb_db'; read x'
 
 
     # For Curve
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC1_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "3017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3017/ -P '$dbC1_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "$dbCn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbCn_y'/ -P '$dbCn_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {
@@ -189,11 +193,11 @@ function launching_scripts {
 helptext;
 
 ############### TESTS ###################
-isredis=`screen -ls | awk '/\.Redis\t/ {print strtonum($1)}'`
-islvldb=`screen -ls | awk '/\.LevelDB\t/ {print strtonum($1)}'`
-islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
-isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
-isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`
+isredis=`screen -ls | egrep '[0-9]+.Redis' | cut -d. -f1`
+islvldb=`screen -ls | egrep '[0-9]+.LevelDB' | cut -d. -f1`
+islogged=`screen -ls | egrep '[0-9]+.Logging' | cut -d. -f1`
+isqueued=`screen -ls | egrep '[0-9]+.Queue' | cut -d. -f1`
+isscripted=`screen -ls | egrep '[0-9]+.Script' | cut -d. -f1`
 
 options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config")
 
@@ -255,6 +259,7 @@ for i in ${!options[@]}; do
             Killall)
                 if [[ $isredis || $islvldb || $islogged || $isqueued || $isscripted ]]; then
                     kill $isredis $islvldb $islogged $isqueued $isscripted
+                    sleep 0.2
                     echo -e $ROSE`screen -ls`$DEFAULT
                     echo -e $GREEN"\t* $isredis $islvldb $islogged $isqueued $isscripted killed."$DEFAULT
                 else

installing_deps.sh

@@ -84,7 +84,11 @@ if [ -z "$VIRTUAL_ENV" ]; then
 
 fi
 
+year1=20`date +%y`
+year2=30`date +%y`
 mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
+mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1
+mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year2
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016
 

pip_packages_requirement.txt

@@ -45,7 +45,7 @@ pycountry
 PySocks
 
 #ASN lookup requirements
-#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
+https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
 https://github.com/trolldbois/python-cymru-services/archive/master.zip
 
 https://github.com/saffsd/langid.py/archive/master.zip

var/www/modules/browsepastes/templates/important_paste_by_module.html

@@ -120,6 +120,10 @@ $("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}");
 
 $(document).ready(function(){
     $('[data-toggle="tooltip"]').tooltip();
+    $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+        //get_html_and_update_modal(event);
+        get_html_and_update_modal(event, $(this));
+    });
 
     search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] });
 
@@ -182,19 +186,12 @@ $(document).ready(function(){
        $("#load-more-button").hide();
    }
 
-
-   // Use to bind the button with the new displayed data
-   // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
-
-   search_table.on( 'draw.dt', function () {
-       // Bind tooltip each time we draw a new page
-       $('[data-toggle="tooltip"]').tooltip();
-       // On click, get html content from url and update the corresponding modal
-       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+   function get_html_and_update_modal(event, truemodal) {
        event.preventDefault();
-           var modal=$(this);
-           var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-           last_clicked_paste =  $(this).attr('data-num');
+
+       var modal=truemodal;
+       var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
+       last_clicked_paste =  modal.attr('data-num');
        $.get(url, function (data) {
        
            // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
@@ -231,6 +228,18 @@ $(document).ready(function(){
                $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
            }
        });
+   }
+
+
+   // Use to bind the button with the new displayed data
+   // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
+
+   search_table.on( 'draw.dt', function () {
+       // Bind tooltip each time we draw a new page
+       $('[data-toggle="tooltip"]').tooltip();
+       // On click, get html content from url and update the corresponding modal
+       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+           get_html_and_update_modal(event, $(this));
        });
    } );
 

var/www/static/js/indexjavascript.js

@@ -91,7 +91,7 @@ function update_values() {
         return all_res;
     }
 
-    var updateInterval = 10000;
+    var updateInterval = 30*1000; //30s = 30*1000ms
     var options_processed_pastes = {
         series: {   shadowSize: 0 ,
                     lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}

var/www/templates/header.html

@@ -1,32 +0,0 @@
-<div class="navbar-header">
-    <ul class="nav navbar-nav">
-	<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-
-	<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-
-	<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-
-	<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-
-	<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
-    <span class="caret"></span></a>
-    <ul class="dropdown-menu">
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
-    </ul>
-</li>
-
-        <li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
-    <span class="caret"></span></a>
-    <ul class="dropdown-menu">
-      <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
-      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
-    </ul>
-</li>
-
-	<li id='page-myNewModule'><a href="{{ url_for('myNewModule.myNewModule_page') }}"><i class="glyphicon glyphicon-new-window"></i> myNewModule </a></li>
-
-    </ul>
-</div>