mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Merged with previous changes for PR
This commit is contained in:
commit
4d59871f00
6 changed files with 55 additions and 11 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -26,3 +26,4 @@ bin/packages/config.cfg
|
||||||
|
|
||||||
# installed files
|
# installed files
|
||||||
nltk_data/
|
nltk_data/
|
||||||
|
doc/all_modules.txt
|
||||||
|
|
|
@ -15,13 +15,14 @@ from pubsublogger import publisher
|
||||||
|
|
||||||
from whoosh.index import create_in, exists_in, open_dir
|
from whoosh.index import create_in, exists_in, open_dir
|
||||||
from whoosh.fields import Schema, TEXT, ID
|
from whoosh.fields import Schema, TEXT, ID
|
||||||
|
import shutil
|
||||||
import os
|
import os
|
||||||
from os.path import join, getsize
|
from os.path import join, getsize
|
||||||
|
|
||||||
from Helper import Process
|
from Helper import Process
|
||||||
|
|
||||||
# Config variable
|
# Config variable
|
||||||
TIME_WAIT = 1.0 #sec
|
TIME_WAIT = 60*15 #sec
|
||||||
|
|
||||||
# return in bytes
|
# return in bytes
|
||||||
def check_index_size(baseindexpath, indexname):
|
def check_index_size(baseindexpath, indexname):
|
||||||
|
@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname):
|
||||||
return cur_sum
|
return cur_sum
|
||||||
|
|
||||||
def move_index_into_old_index_folder(baseindexpath):
|
def move_index_into_old_index_folder(baseindexpath):
|
||||||
command_move = "mv {} {}"
|
for cur_file in os.listdir(baseindexpath):
|
||||||
os.mkdir(join(baseindexpath, "old_index"))
|
if not cur_file == "old_index":
|
||||||
for files in os.listdir(baseindexpath):
|
shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file))
|
||||||
if not files == "old_index":
|
|
||||||
os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files)))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
@ -112,9 +112,17 @@ function launching_queues {
|
||||||
}
|
}
|
||||||
|
|
||||||
function launching_scripts {
|
function launching_scripts {
|
||||||
|
echo -e "\t* Checking configuration"
|
||||||
|
bash -c "./Update-conf.py"
|
||||||
|
exitStatus=$?
|
||||||
|
if [ $exitStatus -ge 1 ]; then
|
||||||
|
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
||||||
|
|
||||||
screen -dmS "Script"
|
screen -dmS "Script"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
|
|
||||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||||
|
|
||||||
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
|
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
|
||||||
|
@ -257,7 +265,15 @@ for i in ${!options[@]}; do
|
||||||
bash -c "./Shutdown.py"
|
bash -c "./Shutdown.py"
|
||||||
;;
|
;;
|
||||||
Update-config)
|
Update-config)
|
||||||
|
echo -e "\t* Checking configuration"
|
||||||
bash -c "./Update-conf.py"
|
bash -c "./Update-conf.py"
|
||||||
|
exitStatus=$?
|
||||||
|
if [ $exitStatus -ge 1 ]; then
|
||||||
|
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
||||||
|
exit
|
||||||
|
else
|
||||||
|
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
||||||
|
fi
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
15
bin/Phone.py
15
bin/Phone.py
|
@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold.
|
||||||
|
|
||||||
import time
|
import time
|
||||||
import re
|
import re
|
||||||
|
import phonenumbers
|
||||||
from packages import Paste
|
from packages import Paste
|
||||||
from pubsublogger import publisher
|
from pubsublogger import publisher
|
||||||
from Helper import Process
|
from Helper import Process
|
||||||
|
@ -35,6 +36,20 @@ def search_phone(message):
|
||||||
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
|
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
|
||||||
#Send to duplicate
|
#Send to duplicate
|
||||||
p.populate_set_out(message, 'Duplicate')
|
p.populate_set_out(message, 'Duplicate')
|
||||||
|
stats = {}
|
||||||
|
for phone_number in results:
|
||||||
|
try:
|
||||||
|
x = phonenumbers.parse(phone_number, None)
|
||||||
|
country_code = x.country_code
|
||||||
|
if stats.get(country_code) is None:
|
||||||
|
stats[country_code] = 1
|
||||||
|
else:
|
||||||
|
stats[country_code] = stats[country_code] + 1
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
for country_code in stats:
|
||||||
|
if stats[country_code] > 4:
|
||||||
|
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||||
|
|
|
@ -5,11 +5,15 @@ import ConfigParser
|
||||||
from ConfigParser import ConfigParser as cfgP
|
from ConfigParser import ConfigParser as cfgP
|
||||||
import os
|
import os
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
import sys
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
|
||||||
|
#return true if the configuration is up-to-date
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||||
|
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
|
||||||
if not os.path.exists(configfile):
|
if not os.path.exists(configfile):
|
||||||
raise Exception('Unable to find the configuration file. \
|
raise Exception('Unable to find the configuration file. \
|
||||||
Did you set environment variables? \
|
Did you set environment variables? \
|
||||||
|
@ -45,8 +49,8 @@ def main():
|
||||||
dicoMissingItem[sec] = list_items
|
dicoMissingItem[sec] = list_items
|
||||||
|
|
||||||
if len(missingSection) == 0 and len(missingItem) == 0:
|
if len(missingSection) == 0 and len(missingItem) == 0:
|
||||||
print("Configuration up-to-date")
|
#print("Configuration up-to-date")
|
||||||
return
|
return True
|
||||||
print("/!\\ Configuration not complete. Missing following configuration: /!\\")
|
print("/!\\ Configuration not complete. Missing following configuration: /!\\")
|
||||||
print("+--------------------------------------------------------------------+")
|
print("+--------------------------------------------------------------------+")
|
||||||
for section in missingSection:
|
for section in missingSection:
|
||||||
|
@ -62,8 +66,12 @@ def main():
|
||||||
resp = raw_input("Do you want to auto fix it? [y/n] ")
|
resp = raw_input("Do you want to auto fix it? [y/n] ")
|
||||||
|
|
||||||
if resp != 'y':
|
if resp != 'y':
|
||||||
return
|
return False
|
||||||
else:
|
else:
|
||||||
|
resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
|
||||||
|
if resp2 == 'y':
|
||||||
|
shutil.move(configfile, configfileBackup)
|
||||||
|
|
||||||
#Do not keep item ordering in section. New items appened
|
#Do not keep item ordering in section. New items appened
|
||||||
for section in missingItem:
|
for section in missingItem:
|
||||||
for item, value in dicoMissingItem[section]:
|
for item, value in dicoMissingItem[section]:
|
||||||
|
@ -75,6 +83,7 @@ def main():
|
||||||
|
|
||||||
with open(configfile, 'w') as f:
|
with open(configfile, 'w') as f:
|
||||||
cfg.write(f)
|
cfg.write(f)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
''' Return a new dico with the section ordered as the old configuration with the updated one added '''
|
''' Return a new dico with the section ordered as the old configuration with the updated one added '''
|
||||||
|
@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
if main():
|
||||||
|
sys.exit()
|
||||||
|
else:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
|
@ -26,6 +26,7 @@ ssdeep
|
||||||
python-magic
|
python-magic
|
||||||
pybloomfiltermmap
|
pybloomfiltermmap
|
||||||
psutil
|
psutil
|
||||||
|
phonenumbers
|
||||||
|
|
||||||
ipython
|
ipython
|
||||||
flask
|
flask
|
||||||
|
|
Loading…
Reference in a new issue