mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Merged with previous changes for PR
This commit is contained in:
commit
4d59871f00
6 changed files with 55 additions and 11 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -26,3 +26,4 @@ bin/packages/config.cfg
|
|||
|
||||
# installed files
|
||||
nltk_data/
|
||||
doc/all_modules.txt
|
||||
|
|
|
@ -15,13 +15,14 @@ from pubsublogger import publisher
|
|||
|
||||
from whoosh.index import create_in, exists_in, open_dir
|
||||
from whoosh.fields import Schema, TEXT, ID
|
||||
import shutil
|
||||
import os
|
||||
from os.path import join, getsize
|
||||
|
||||
from Helper import Process
|
||||
|
||||
# Config variable
|
||||
TIME_WAIT = 1.0 #sec
|
||||
TIME_WAIT = 60*15 #sec
|
||||
|
||||
# return in bytes
|
||||
def check_index_size(baseindexpath, indexname):
|
||||
|
@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname):
|
|||
return cur_sum
|
||||
|
||||
def move_index_into_old_index_folder(baseindexpath):
|
||||
command_move = "mv {} {}"
|
||||
os.mkdir(join(baseindexpath, "old_index"))
|
||||
for files in os.listdir(baseindexpath):
|
||||
if not files == "old_index":
|
||||
os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files)))
|
||||
for cur_file in os.listdir(baseindexpath):
|
||||
if not cur_file == "old_index":
|
||||
shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
@ -112,9 +112,17 @@ function launching_queues {
|
|||
}
|
||||
|
||||
function launching_scripts {
|
||||
echo -e "\t* Checking configuration"
|
||||
bash -c "./Update-conf.py"
|
||||
exitStatus=$?
|
||||
if [ $exitStatus -ge 1 ]; then
|
||||
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
||||
exit
|
||||
fi
|
||||
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
||||
|
||||
screen -dmS "Script"
|
||||
sleep 0.1
|
||||
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
|
||||
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
|
||||
|
@ -257,7 +265,15 @@ for i in ${!options[@]}; do
|
|||
bash -c "./Shutdown.py"
|
||||
;;
|
||||
Update-config)
|
||||
echo -e "\t* Checking configuration"
|
||||
bash -c "./Update-conf.py"
|
||||
exitStatus=$?
|
||||
if [ $exitStatus -ge 1 ]; then
|
||||
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
||||
exit
|
||||
else
|
||||
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
|
15
bin/Phone.py
15
bin/Phone.py
|
@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold.
|
|||
|
||||
import time
|
||||
import re
|
||||
import phonenumbers
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
@ -35,6 +36,20 @@ def search_phone(message):
|
|||
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
stats = {}
|
||||
for phone_number in results:
|
||||
try:
|
||||
x = phonenumbers.parse(phone_number, None)
|
||||
country_code = x.country_code
|
||||
if stats.get(country_code) is None:
|
||||
stats[country_code] = 1
|
||||
else:
|
||||
stats[country_code] = stats[country_code] + 1
|
||||
except:
|
||||
pass
|
||||
for country_code in stats:
|
||||
if stats[country_code] > 4:
|
||||
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
|
|
|
@ -5,11 +5,15 @@ import ConfigParser
|
|||
from ConfigParser import ConfigParser as cfgP
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
|
||||
#return true if the configuration is up-to-date
|
||||
def main():
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
|
@ -45,8 +49,8 @@ def main():
|
|||
dicoMissingItem[sec] = list_items
|
||||
|
||||
if len(missingSection) == 0 and len(missingItem) == 0:
|
||||
print("Configuration up-to-date")
|
||||
return
|
||||
#print("Configuration up-to-date")
|
||||
return True
|
||||
print("/!\\ Configuration not complete. Missing following configuration: /!\\")
|
||||
print("+--------------------------------------------------------------------+")
|
||||
for section in missingSection:
|
||||
|
@ -62,8 +66,12 @@ def main():
|
|||
resp = raw_input("Do you want to auto fix it? [y/n] ")
|
||||
|
||||
if resp != 'y':
|
||||
return
|
||||
return False
|
||||
else:
|
||||
resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
|
||||
if resp2 == 'y':
|
||||
shutil.move(configfile, configfileBackup)
|
||||
|
||||
#Do not keep item ordering in section. New items appened
|
||||
for section in missingItem:
|
||||
for item, value in dicoMissingItem[section]:
|
||||
|
@ -75,6 +83,7 @@ def main():
|
|||
|
||||
with open(configfile, 'w') as f:
|
||||
cfg.write(f)
|
||||
return True
|
||||
|
||||
|
||||
''' Return a new dico with the section ordered as the old configuration with the updated one added '''
|
||||
|
@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
if main():
|
||||
sys.exit()
|
||||
else:
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
@ -26,6 +26,7 @@ ssdeep
|
|||
python-magic
|
||||
pybloomfiltermmap
|
||||
psutil
|
||||
phonenumbers
|
||||
|
||||
ipython
|
||||
flask
|
||||
|
|
Loading…
Reference in a new issue