Merge branch 'bugfix'

.gitignore

@@ -20,6 +20,7 @@ var/www/static/
 !var/www/static/js/moduleTrending.js
 !var/www/static/js/plot-graph.js
 !var/www/static/js/trendingchart.js
+var/www/templates/header.html # auto-generated
 
 # Local config
 bin/packages/config.cfg
@@ -27,3 +28,4 @@ bin/packages/config.cfg
 # installed files
 nltk_data/
 doc/all_modules.txt
+doc/module-data-flow.png # auto-generated

bin/LAUNCH.sh

@@ -66,32 +66,36 @@ function launching_redis {
 
 function launching_lvldb {
     #Want to launch more level_db?
+    #FIXME update the date in config.cfg
     lvdbhost='127.0.0.1'
     lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
-    db1_y='2013'
+    db1_y='2016'
-    db2_y='2014'
+    db2_y='2017'
-    db3_y='2016'
+    dbn_y=`date +%Y`
-    db4_y='2017'
 
-    dbC_y='3016'
+    dbC1_y='3016'
+    dbCn_y=30`date +%y`
     nb_db=13
 
     screen -dmS "LevelDB"
     sleep 0.1
     echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
     #Add lines here with appropriates options.
-    screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db2_y' -M '$nb_db'; read x'
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "$dbn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbn_y'/ -P '$dbn_y' -M '$nb_db'; read x'
 
 
     # For Curve
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC1_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "3017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3017/ -P '$dbC1_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "$dbCn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbCn_y'/ -P '$dbCn_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {
@@ -189,11 +193,11 @@ function launching_scripts {
 helptext;
 
 ############### TESTS ###################
-isredis=`screen -ls | awk '/\.Redis\t/ {print strtonum($1)}'`
+isredis=`screen -ls | egrep '[0-9]+.Redis' | cut -d. -f1`
-islvldb=`screen -ls | awk '/\.LevelDB\t/ {print strtonum($1)}'`
+islvldb=`screen -ls | egrep '[0-9]+.LevelDB' | cut -d. -f1`
-islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
+islogged=`screen -ls | egrep '[0-9]+.Logging' | cut -d. -f1`
-isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
+isqueued=`screen -ls | egrep '[0-9]+.Queue' | cut -d. -f1`
-isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`
+isscripted=`screen -ls | egrep '[0-9]+.Script' | cut -d. -f1`
 
 options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config")
 
@@ -255,6 +259,7 @@ for i in ${!options[@]}; do
             Killall)
                 if [[ $isredis || $islvldb || $islogged || $isqueued || $isscripted ]]; then
                     kill $isredis $islvldb $islogged $isqueued $isscripted
+                    sleep 0.2
                     echo -e $ROSE`screen -ls`$DEFAULT
                     echo -e $GREEN"\t* $isredis $islvldb $islogged $isqueued $isscripted killed."$DEFAULT
                 else

installing_deps.sh

@@ -84,7 +84,11 @@ if [ -z "$VIRTUAL_ENV" ]; then
 
 fi
 
+year1=20`date +%y`
+year2=30`date +%y`
 mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
+mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1
+mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year2
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016
 

pip_packages_requirement.txt

@@ -45,7 +45,7 @@ pycountry
 PySocks
 
 #ASN lookup requirements
-#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
+https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
 https://github.com/trolldbois/python-cymru-services/archive/master.zip
 
 https://github.com/saffsd/langid.py/archive/master.zip

var/www/modules/browsepastes/templates/important_paste_by_module.html

@@ -120,6 +120,10 @@ $("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}");
 
 $(document).ready(function(){
     $('[data-toggle="tooltip"]').tooltip();
+    $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+        //get_html_and_update_modal(event);
+        get_html_and_update_modal(event, $(this));
+    });
 
     search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] });
 
@@ -182,6 +186,50 @@ $(document).ready(function(){
        $("#load-more-button").hide();
    }
 
+   function get_html_and_update_modal(event, truemodal) {
+       event.preventDefault();
+
+       var modal=truemodal;
+       var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
+       last_clicked_paste =  modal.attr('data-num');
+       $.get(url, function (data) {
+       
+           // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
+           var received_num = parseInt(data.split("|num|")[1]);
+           if (received_num == last_clicked_paste && can_change_modal_content) {
+               can_change_modal_content = false;
+       
+               // clear data by removing html, body, head tags. prevent dark modal background stack bug.
+               var cleared_data = data.split("<body>")[1].split("</body>")[0];
+               $("#mymodalbody").html(cleared_data);
+       
+               var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
+               button.tooltip();
+               $("#mymodalbody").children(".panel-default").append(button);
+       
+               $("#button_show_path").attr('href', $(modal).attr('data-url'));
+               $("#button_show_path").show('fast');
+               $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+               if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                   nothing_to_display();
+               }
+               // On click, donwload all paste's content
+               $("#load-more-button").on("click", function (event) {
+                   if (complete_paste == null) { //Donwload only once
+                       $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                           complete_paste = data;
+                           update_preview();
+                       });
+                   } else {
+                       update_preview();
+                   }
+               });
+           } else if (can_change_modal_content) {
+               $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+           }
+       });
+   }
+
 
    // Use to bind the button with the new displayed data
    // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
@@ -191,46 +239,7 @@ $(document).ready(function(){
        $('[data-toggle="tooltip"]').tooltip();
        // On click, get html content from url and update the corresponding modal
        $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
-           event.preventDefault();
+           get_html_and_update_modal(event, $(this));
-           var modal=$(this);
-           var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-           last_clicked_paste =  $(this).attr('data-num');
-           $.get(url, function (data) {
-
-               // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
-               var received_num = parseInt(data.split("|num|")[1]);
-               if (received_num == last_clicked_paste && can_change_modal_content) {
-                   can_change_modal_content = false;
-
-                   // clear data by removing html, body, head tags. prevent dark modal background stack bug.
-                   var cleared_data = data.split("<body>")[1].split("</body>")[0];
-                   $("#mymodalbody").html(cleared_data);
-
-                   var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-                   button.tooltip();
-                   $("#mymodalbody").children(".panel-default").append(button);
-       
-                   $("#button_show_path").attr('href', $(modal).attr('data-url'));
-                   $("#button_show_path").show('fast');
-                   $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-                   if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                       nothing_to_display();
-                   }
-                   // On click, donwload all paste's content
-                   $("#load-more-button").on("click", function (event) {
-                       if (complete_paste == null) { //Donwload only once
-                           $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                               complete_paste = data;
-                               update_preview();
-                           });
-                       } else {
-                           update_preview();
-                       }
-                   });
-               } else if (can_change_modal_content) {
-                   $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
-               }
-           });
        });
    } );
 

var/www/static/js/indexjavascript.js

@@ -91,7 +91,7 @@ function update_values() {
         return all_res;
     }
 
-    var updateInterval = 10000;
+    var updateInterval = 30*1000; //30s = 30*1000ms
     var options_processed_pastes = {
         series: {   shadowSize: 0 ,
                     lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}

var/www/templates/header.html

@@ -1,32 +0,0 @@
-<div class="navbar-header">
-    <ul class="nav navbar-nav">
-	<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-
-	<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-
-	<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-
-	<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-
-	<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
-    <span class="caret"></span></a>
-    <ul class="dropdown-menu">
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
-    </ul>
-</li>
-
-        <li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
-    <span class="caret"></span></a>
-    <ul class="dropdown-menu">
-      <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
-      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
-    </ul>
-</li>
-
-	<li id='page-myNewModule'><a href="{{ url_for('myNewModule.myNewModule_page') }}"><i class="glyphicon glyphicon-new-window"></i> myNewModule </a></li>
-
-    </ul>
-</div>