Merged with previous changes for PR

.gitignore

@@ -26,3 +26,4 @@ bin/packages/config.cfg
 
 # installed files
 nltk_data/
+doc/all_modules.txt

bin/Indexer.py

@@ -15,13 +15,14 @@ from pubsublogger import publisher
 
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.fields import Schema, TEXT, ID
+import shutil
 import os
 from os.path import join, getsize
 
 from Helper import Process
 
 # Config variable
-TIME_WAIT = 1.0 #sec
+TIME_WAIT = 60*15 #sec
 
 # return in bytes
 def check_index_size(baseindexpath, indexname):
@@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname):
     return cur_sum
 
 def move_index_into_old_index_folder(baseindexpath):
-    command_move = "mv {} {}"
-    os.mkdir(join(baseindexpath, "old_index"))
-    for files in os.listdir(baseindexpath):
-        if not files == "old_index":
-            os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files)))
+    for cur_file in os.listdir(baseindexpath):
+        if not cur_file == "old_index":
+            shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file))
 
 
 if __name__ == "__main__":

bin/LAUNCH.sh

@@ -112,9 +112,17 @@ function launching_queues {
 }
 
 function launching_scripts {
+    echo -e "\t* Checking configuration"
+    bash -c "./Update-conf.py"
+    exitStatus=$?
+    if [ $exitStatus -ge 1 ]; then
+        echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+        exit
+    fi
+    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+
     screen -dmS "Script"
     sleep 0.1
-
     echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
     screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
@@ -257,7 +265,15 @@ for i in ${!options[@]}; do
                 bash -c "./Shutdown.py"
                 ;;
             Update-config)
+                echo -e "\t* Checking configuration"
                 bash -c "./Update-conf.py"
+                exitStatus=$?
+                if [ $exitStatus -ge 1 ]; then
+                    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+                    exit
+                else
+                    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+                fi
                 ;;
         esac
     fi

bin/Phone.py

@@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold.
 
 import time
 import re
+import phonenumbers
 from packages import Paste
 from pubsublogger import publisher
 from Helper import Process
@@ -35,6 +36,20 @@ def search_phone(message):
         p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
         #Send to duplicate
         p.populate_set_out(message, 'Duplicate')
+        stats = {}
+        for phone_number in results:
+            try:
+                x = phonenumbers.parse(phone_number, None)
+                country_code = x.country_code
+                if stats.get(country_code) is None:
+                    stats[country_code] = 1
+                else:
+                    stats[country_code] = stats[country_code] + 1
+            except:
+                pass
+        for country_code in stats:
+            if stats[country_code] > 4:
+                publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)

bin/Update-conf.py

@@ -5,11 +5,15 @@ import ConfigParser
 from ConfigParser import ConfigParser as cfgP
 import os
 from collections import OrderedDict
+import sys
+import shutil
 
 
+#return true if the configuration is up-to-date
 def main():
 
     configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
     if not os.path.exists(configfile):
         raise Exception('Unable to find the configuration file. \
                         Did you set environment variables? \
@@ -45,8 +49,8 @@ def main():
                 dicoMissingItem[sec] = list_items
 
     if len(missingSection) == 0 and len(missingItem) == 0:
-        print("Configuration up-to-date")
-        return
+        #print("Configuration up-to-date")
+        return True
     print("/!\\ Configuration not complete. Missing following configuration: /!\\")
     print("+--------------------------------------------------------------------+")
     for section in missingSection:
@@ -62,8 +66,12 @@ def main():
     resp = raw_input("Do you want to auto fix it? [y/n] ")
 
     if resp != 'y':
-        return
+        return False
     else:
+        resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
+        if resp2 == 'y':
+            shutil.move(configfile, configfileBackup)
+
         #Do not keep item ordering in section. New items appened
         for section in missingItem:
             for item, value in dicoMissingItem[section]:
@@ -75,6 +83,7 @@ def main():
 
         with open(configfile, 'w') as f:
             cfg.write(f)
+        return True
 
 
 ''' Return a new dico with the section ordered as the old configuration with the updated one added '''
@@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis
 
 
 if __name__ == "__main__":
-    main()
+    if main():
+        sys.exit()
+    else:
+        sys.exit(1)
 

pip_packages_requirement.txt

@@ -26,6 +26,7 @@ ssdeep
 python-magic
 pybloomfiltermmap
 psutil
+phonenumbers
 
 ipython
 flask