Merged with previous changes for PR

This commit is contained in:
Mokaddem 2017-07-17 13:48:16 +02:00
commit 4d59871f00
6 changed files with 55 additions and 11 deletions

1
.gitignore vendored
View file

@ -26,3 +26,4 @@ bin/packages/config.cfg
# installed files
nltk_data/
doc/all_modules.txt

View file

@ -15,13 +15,14 @@ from pubsublogger import publisher
from whoosh.index import create_in, exists_in, open_dir
from whoosh.fields import Schema, TEXT, ID
import shutil
import os
from os.path import join, getsize
from Helper import Process
# Config variable
TIME_WAIT = 1.0 #sec
TIME_WAIT = 60*15 #sec
# return in bytes
def check_index_size(baseindexpath, indexname):
@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname):
return cur_sum
def move_index_into_old_index_folder(baseindexpath):
command_move = "mv {} {}"
os.mkdir(join(baseindexpath, "old_index"))
for files in os.listdir(baseindexpath):
if not files == "old_index":
os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files)))
for cur_file in os.listdir(baseindexpath):
if not cur_file == "old_index":
shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file))
if __name__ == "__main__":

View file

@ -112,9 +112,17 @@ function launching_queues {
}
function launching_scripts {
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
fi
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
screen -dmS "Script"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
@ -257,7 +265,15 @@ for i in ${!options[@]}; do
bash -c "./Shutdown.py"
;;
Update-config)
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
fi
;;
esac
fi

View file

@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold.
import time
import re
import phonenumbers
from packages import Paste
from pubsublogger import publisher
from Helper import Process
@ -35,6 +36,20 @@ def search_phone(message):
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)

View file

@ -5,11 +5,15 @@ import ConfigParser
from ConfigParser import ConfigParser as cfgP
import os
from collections import OrderedDict
import sys
import shutil
#return true if the configuration is up-to-date
def main():
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
@ -45,8 +49,8 @@ def main():
dicoMissingItem[sec] = list_items
if len(missingSection) == 0 and len(missingItem) == 0:
print("Configuration up-to-date")
return
#print("Configuration up-to-date")
return True
print("/!\\ Configuration not complete. Missing following configuration: /!\\")
print("+--------------------------------------------------------------------+")
for section in missingSection:
@ -62,8 +66,12 @@ def main():
resp = raw_input("Do you want to auto fix it? [y/n] ")
if resp != 'y':
return
return False
else:
resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
if resp2 == 'y':
shutil.move(configfile, configfileBackup)
#Do not keep item ordering in section. New items appened
for section in missingItem:
for item, value in dicoMissingItem[section]:
@ -75,6 +83,7 @@ def main():
with open(configfile, 'w') as f:
cfg.write(f)
return True
''' Return a new dico with the section ordered as the old configuration with the updated one added '''
@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis
if __name__ == "__main__":
main()
if main():
sys.exit()
else:
sys.exit(1)

View file

@ -26,6 +26,7 @@ ssdeep
python-magic
pybloomfiltermmap
psutil
phonenumbers
ipython
flask