Merged with previous changes for PR

This commit is contained in:
Mokaddem 2017-07-17 13:48:16 +02:00
commit 4d59871f00
6 changed files with 55 additions and 11 deletions

1
.gitignore vendored
View file

@ -26,3 +26,4 @@ bin/packages/config.cfg
# installed files # installed files
nltk_data/ nltk_data/
doc/all_modules.txt

View file

@ -15,13 +15,14 @@ from pubsublogger import publisher
from whoosh.index import create_in, exists_in, open_dir from whoosh.index import create_in, exists_in, open_dir
from whoosh.fields import Schema, TEXT, ID from whoosh.fields import Schema, TEXT, ID
import shutil
import os import os
from os.path import join, getsize from os.path import join, getsize
from Helper import Process from Helper import Process
# Config variable # Config variable
TIME_WAIT = 1.0 #sec TIME_WAIT = 60*15 #sec
# return in bytes # return in bytes
def check_index_size(baseindexpath, indexname): def check_index_size(baseindexpath, indexname):
@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname):
return cur_sum return cur_sum
def move_index_into_old_index_folder(baseindexpath): def move_index_into_old_index_folder(baseindexpath):
command_move = "mv {} {}" for cur_file in os.listdir(baseindexpath):
os.mkdir(join(baseindexpath, "old_index")) if not cur_file == "old_index":
for files in os.listdir(baseindexpath): shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file))
if not files == "old_index":
os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files)))
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -112,9 +112,17 @@ function launching_queues {
} }
function launching_scripts { function launching_scripts {
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
fi
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
screen -dmS "Script" screen -dmS "Script"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
@ -257,7 +265,15 @@ for i in ${!options[@]}; do
bash -c "./Shutdown.py" bash -c "./Shutdown.py"
;; ;;
Update-config) Update-config)
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py" bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
fi
;; ;;
esac esac
fi fi

View file

@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold.
import time import time
import re import re
import phonenumbers
from packages import Paste from packages import Paste
from pubsublogger import publisher from pubsublogger import publisher
from Helper import Process from Helper import Process
@ -35,6 +36,20 @@ def search_phone(message):
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
#Send to duplicate #Send to duplicate
p.populate_set_out(message, 'Duplicate') p.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
if __name__ == '__main__': if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)

View file

@ -5,11 +5,15 @@ import ConfigParser
from ConfigParser import ConfigParser as cfgP from ConfigParser import ConfigParser as cfgP
import os import os
from collections import OrderedDict from collections import OrderedDict
import sys
import shutil
#return true if the configuration is up-to-date
def main(): def main():
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
if not os.path.exists(configfile): if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \ raise Exception('Unable to find the configuration file. \
Did you set environment variables? \ Did you set environment variables? \
@ -45,8 +49,8 @@ def main():
dicoMissingItem[sec] = list_items dicoMissingItem[sec] = list_items
if len(missingSection) == 0 and len(missingItem) == 0: if len(missingSection) == 0 and len(missingItem) == 0:
print("Configuration up-to-date") #print("Configuration up-to-date")
return return True
print("/!\\ Configuration not complete. Missing following configuration: /!\\") print("/!\\ Configuration not complete. Missing following configuration: /!\\")
print("+--------------------------------------------------------------------+") print("+--------------------------------------------------------------------+")
for section in missingSection: for section in missingSection:
@ -62,8 +66,12 @@ def main():
resp = raw_input("Do you want to auto fix it? [y/n] ") resp = raw_input("Do you want to auto fix it? [y/n] ")
if resp != 'y': if resp != 'y':
return return False
else: else:
resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
if resp2 == 'y':
shutil.move(configfile, configfileBackup)
#Do not keep item ordering in section. New items appened #Do not keep item ordering in section. New items appened
for section in missingItem: for section in missingItem:
for item, value in dicoMissingItem[section]: for item, value in dicoMissingItem[section]:
@ -75,6 +83,7 @@ def main():
with open(configfile, 'w') as f: with open(configfile, 'w') as f:
cfg.write(f) cfg.write(f)
return True
''' Return a new dico with the section ordered as the old configuration with the updated one added ''' ''' Return a new dico with the section ordered as the old configuration with the updated one added '''
@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis
if __name__ == "__main__": if __name__ == "__main__":
main() if main():
sys.exit()
else:
sys.exit(1)

View file

@ -26,6 +26,7 @@ ssdeep
python-magic python-magic
pybloomfiltermmap pybloomfiltermmap
psutil psutil
phonenumbers
ipython ipython
flask flask