diff --git a/.gitignore b/.gitignore index a99c47fd..d62fbd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ var/www/static/ !var/www/static/js/moduleTrending.js !var/www/static/js/plot-graph.js !var/www/static/js/trendingchart.js +var/www/templates/header.html # auto-generated # Local config bin/packages/config.cfg @@ -27,3 +28,4 @@ bin/packages/config.cfg # installed files nltk_data/ doc/all_modules.txt +doc/module-data-flow.png # auto-generated diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 38ee4e83..f33af903 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,32 +66,36 @@ function launching_redis { function launching_lvldb { #Want to launch more level_db? + #FIXME update the date in config.cfg lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" - db1_y='2013' - db2_y='2014' - db3_y='2016' - db4_y='2017' + db1_y='2016' + db2_y='2017' + dbn_y=`date +%Y` - dbC_y='3016' + dbC1_y='3016' + dbCn_y=30`date +%y` nb_db=13 screen -dmS "LevelDB" sleep 0.1 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT #Add lines here with appropriates options. - screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' sleep 0.1 - screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' + screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db1_y' -M '$nb_db'; read x' sleep 0.1 - screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' + screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db2_y' -M '$nb_db'; read x' sleep 0.1 - screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x' + screen -S "LevelDB" -X screen -t "$dbn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbn_y'/ -P '$dbn_y' -M '$nb_db'; read x' # For Curve sleep 0.1 - screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x' + screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC1_y' -M '$nb_db'; read x' + sleep 0.1 + screen -S "LevelDB" -X screen -t "3017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3017/ -P '$dbC1_y' -M '$nb_db'; read x' + sleep 0.1 + screen -S "LevelDB" -X screen -t "$dbCn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbCn_y'/ -P '$dbCn_y' -M '$nb_db'; read x' } function launching_logs { @@ -189,11 +193,11 @@ function launching_scripts { helptext; ############### TESTS ################### -isredis=`screen -ls | awk '/\.Redis\t/ {print strtonum($1)}'` -islvldb=`screen -ls | awk '/\.LevelDB\t/ {print strtonum($1)}'` -islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'` -isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'` -isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'` +isredis=`screen -ls | egrep '[0-9]+.Redis' | cut -d. -f1` +islvldb=`screen -ls | egrep '[0-9]+.LevelDB' | cut -d. -f1` +islogged=`screen -ls | egrep '[0-9]+.Logging' | cut -d. -f1` +isqueued=`screen -ls | egrep '[0-9]+.Queue' | cut -d. -f1` +isscripted=`screen -ls | egrep '[0-9]+.Script' | cut -d. -f1` options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config") @@ -255,6 +259,7 @@ for i in ${!options[@]}; do Killall) if [[ $isredis || $islvldb || $islogged || $isqueued || $isscripted ]]; then kill $isredis $islvldb $islogged $isqueued $isscripted + sleep 0.2 echo -e $ROSE`screen -ls`$DEFAULT echo -e $GREEN"\t* $isredis $islvldb $islogged $isqueued $isscripted killed."$DEFAULT else diff --git a/doc/module-data-flow.png b/doc/module-data-flow.png deleted file mode 100644 index 03fd7123..00000000 Binary files a/doc/module-data-flow.png and /dev/null differ diff --git a/installing_deps.sh b/installing_deps.sh index ec11f9f2..9f858810 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -84,7 +84,11 @@ if [ -z "$VIRTUAL_ENV" ]; then fi +year1=20`date +%y` +year2=30`date +%y` mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} +mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1 +mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year2 mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016 mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016 diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt index 8c6e956b..d6dd108d 100644 --- a/pip_packages_requirement.txt +++ b/pip_packages_requirement.txt @@ -45,7 +45,7 @@ pycountry PySocks #ASN lookup requirements -#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz +https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz https://github.com/trolldbois/python-cymru-services/archive/master.zip https://github.com/saffsd/langid.py/archive/master.zip diff --git a/var/www/modules/browsepastes/templates/important_paste_by_module.html b/var/www/modules/browsepastes/templates/important_paste_by_module.html index 14048bbb..15fec49a 100644 --- a/var/www/modules/browsepastes/templates/important_paste_by_module.html +++ b/var/www/modules/browsepastes/templates/important_paste_by_module.html @@ -120,6 +120,10 @@ $("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}"); $(document).ready(function(){ $('[data-toggle="tooltip"]').tooltip(); + $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) { + //get_html_and_update_modal(event); + get_html_and_update_modal(event, $(this)); + }); search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] }); @@ -182,6 +186,50 @@ $(document).ready(function(){ $("#load-more-button").hide(); } + function get_html_and_update_modal(event, truemodal) { + event.preventDefault(); + + var modal=truemodal; + var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num'); + last_clicked_paste = modal.attr('data-num'); + $.get(url, function (data) { + + // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. + var received_num = parseInt(data.split("|num|")[1]); + if (received_num == last_clicked_paste && can_change_modal_content) { + can_change_modal_content = false; + + // clear data by removing html, body, head tags. prevent dark modal background stack bug. + var cleared_data = data.split("")[1].split("")[0]; + $("#mymodalbody").html(cleared_data); + + var button = $(''); + button.tooltip(); + $("#mymodalbody").children(".panel-default").append(button); + + $("#button_show_path").attr('href', $(modal).attr('data-url')); + $("#button_show_path").show('fast'); + $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF + if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed + nothing_to_display(); + } + // On click, donwload all paste's content + $("#load-more-button").on("click", function (event) { + if (complete_paste == null) { //Donwload only once + $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ + complete_paste = data; + update_preview(); + }); + } else { + update_preview(); + } + }); + } else if (can_change_modal_content) { + $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num); + } + }); + } + // Use to bind the button with the new displayed data // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page) @@ -191,46 +239,7 @@ $(document).ready(function(){ $('[data-toggle="tooltip"]').tooltip(); // On click, get html content from url and update the corresponding modal $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) { - event.preventDefault(); - var modal=$(this); - var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num'); - last_clicked_paste = $(this).attr('data-num'); - $.get(url, function (data) { - - // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. - var received_num = parseInt(data.split("|num|")[1]); - if (received_num == last_clicked_paste && can_change_modal_content) { - can_change_modal_content = false; - - // clear data by removing html, body, head tags. prevent dark modal background stack bug. - var cleared_data = data.split("")[1].split("")[0]; - $("#mymodalbody").html(cleared_data); - - var button = $(''); - button.tooltip(); - $("#mymodalbody").children(".panel-default").append(button); - - $("#button_show_path").attr('href', $(modal).attr('data-url')); - $("#button_show_path").show('fast'); - $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF - if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed - nothing_to_display(); - } - // On click, donwload all paste's content - $("#load-more-button").on("click", function (event) { - if (complete_paste == null) { //Donwload only once - $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ - complete_paste = data; - update_preview(); - }); - } else { - update_preview(); - } - }); - } else if (can_change_modal_content) { - $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num); - } - }); + get_html_and_update_modal(event, $(this)); }); } ); diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 63ff50a8..47013634 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -91,7 +91,7 @@ function update_values() { return all_res; } - var updateInterval = 10000; + var updateInterval = 30*1000; //30s = 30*1000ms var options_processed_pastes = { series: { shadowSize: 0 , lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }} diff --git a/var/www/templates/header.html b/var/www/templates/header.html deleted file mode 100644 index 1df983a0..00000000 --- a/var/www/templates/header.html +++ /dev/null @@ -1,32 +0,0 @@ -