mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [Update v2.7] add update v2.7 scripts
This commit is contained in:
parent
3a691a4af3
commit
ce3d98d620
7 changed files with 232 additions and 3 deletions
|
@ -79,6 +79,9 @@ class Date(object):
|
|||
comp_day = str(computed_date.day).zfill(2)
|
||||
return comp_year + comp_month + comp_day
|
||||
|
||||
def get_today_date_str():
|
||||
return datetime.date.today().strftime("%Y%m%d")
|
||||
|
||||
def date_add_day(date, num_day=1):
|
||||
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
|
||||
new_date = str(new_date).replace('-', '')
|
||||
|
|
|
@ -86,11 +86,31 @@ if __name__ == "__main__":
|
|||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py')
|
||||
process = subprocess.run(['python' ,update_file])
|
||||
|
||||
update_progress = r_serv_db.get('ail:current_background_script_stat')
|
||||
update_progress = r_serv.get('ail:current_background_script_stat')
|
||||
if update_progress:
|
||||
if int(update_progress) == 100:
|
||||
r_serv.delete('ail:update_in_progress')
|
||||
r_serv.delete('ail:current_background_script')
|
||||
r_serv.delete('ail:current_background_script_stat')
|
||||
r_serv.delete('ail:current_background_update')
|
||||
r_serv_db.srem('ail:to_update', new_version)
|
||||
r_serv.srem('ail:to_update', new_version)
|
||||
|
||||
elif r_serv.sismember('ail:to_update', 'v2.7'):
|
||||
new_version = 'v2.7'
|
||||
r_serv.delete('ail:update_error')
|
||||
r_serv.delete('ail:current_background_script_stat')
|
||||
r_serv.set('ail:update_in_progress', new_version)
|
||||
r_serv.set('ail:current_background_update', new_version)
|
||||
r_serv.set('ail:current_background_script', 'domain tags update')
|
||||
|
||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py')
|
||||
process = subprocess.run(['python' ,update_file])
|
||||
|
||||
update_progress = r_serv.get('ail:current_background_script_stat')
|
||||
if update_progress:
|
||||
if int(update_progress) == 100:
|
||||
r_serv.delete('ail:update_in_progress')
|
||||
r_serv.delete('ail:current_background_script')
|
||||
r_serv.delete('ail:current_background_script_stat')
|
||||
r_serv.delete('ail:current_background_update')
|
||||
r_serv.srem('ail:to_update', new_version)
|
||||
|
|
|
@ -65,6 +65,9 @@ if __name__ == '__main__':
|
|||
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
|
||||
nb_tags_to_update += r_serv_tag.scard(tag)
|
||||
|
||||
if nb_tags_to_update == 0:
|
||||
nb_tags_to_update = 1
|
||||
|
||||
for tag in tags_list:
|
||||
|
||||
all_item = r_serv_tag.smembers(tag)
|
||||
|
|
43
update/v2.7/Update.py
Executable file
43
update/v2.7/Update.py
Executable file
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
new_version = 'v2.7'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
#Set current update_in_progress
|
||||
r_serv.set('ail:update_in_progress', new_version)
|
||||
r_serv.set('ail:current_background_update', new_version)
|
||||
|
||||
r_serv.sadd('ail:to_update', new_version)
|
||||
|
||||
#### Update tags ####
|
||||
r_serv_tags.sunionstore('list_tags:item', 'list_tags', [])
|
||||
r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', [])
|
||||
r_serv_onion.delete('incorrect_domain')
|
||||
r_serv.set('ail:update_v2.7:deletetagrange', 1)
|
||||
#### ####
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
39
update/v2.7/Update.sh
Executable file
39
update/v2.7/Update.sh
Executable file
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.7/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
119
update/v2.7/Update_domain_tags.py
Executable file
119
update/v2.7/Update_domain_tags.py
Executable file
|
@ -0,0 +1,119 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tag
|
||||
|
||||
def sanitize_domain(domain):
|
||||
faup.decode(domain)
|
||||
domain_sanitized = faup.get()
|
||||
domain_sanitized = domain_sanitized['domain']
|
||||
try:
|
||||
domain_sanitized = domain_sanitized.decode()
|
||||
except:
|
||||
pass
|
||||
return domain_sanitized.lower()
|
||||
|
||||
def delete_domain_tag_daterange():
|
||||
all_domains_tags = Tag.get_all_obj_tags('domain')
|
||||
nb_updated = 0
|
||||
nb_to_update = len(all_domains_tags)
|
||||
if nb_to_update == 0:
|
||||
nb_to_update = 1
|
||||
refresh_time = time.time()
|
||||
l_dates = Date.substract_date('20191008', Date.get_today_date_str())
|
||||
for tag in all_domains_tags:
|
||||
for date_day in l_dates:
|
||||
r_serv_tags.delete('domain:{}:{}'.format(tag, date_day))
|
||||
nb_updated += 1
|
||||
refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
|
||||
r_serv_db.delete('ail:update_v2.7:deletetagrange')
|
||||
|
||||
def update_domain_tags(domain):
|
||||
domain_sanitized = sanitize_domain(domain)
|
||||
if domain != domain_sanitized:
|
||||
r_serv_onion.sadd('incorrect_domain', domain)
|
||||
domain = domain_sanitized
|
||||
|
||||
domain_tags = Tag.get_obj_tag(domain)
|
||||
for tag in domain_tags:
|
||||
# delete incorrect tags
|
||||
if tag == 'infoleak:submission="crawler"' and tag == 'infoleak:submission="manual"':
|
||||
r_serv_metadata.srem('tag:{}'.format(domain), tag)
|
||||
else:
|
||||
Tag.add_global_tag(tag, object_type='domain')
|
||||
r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
|
||||
|
||||
def update_progress(refresh_time, nb_updated, nb_elem_to_update):
|
||||
if time.time() - refresh_time > 10:
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
refresh_time = time.time()
|
||||
|
||||
return refresh_time
|
||||
|
||||
def update_db():
|
||||
nb_updated = 0
|
||||
nb_to_update = r_serv_onion.scard('domain_update_v2.7')
|
||||
refresh_time = time.time()
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'domain tags update')
|
||||
domain = r_serv_onion.spop('domain_update_v2.7')
|
||||
while domain is not None:
|
||||
update_domain_tags(domain)
|
||||
nb_updated += 1
|
||||
refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
|
||||
domain = r_serv_onion.spop('domain_update_v2.7')
|
||||
if r_serv_db.exists('ail:update_v2.7:deletetagrange'):
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys')
|
||||
delete_domain_tag_daterange()
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
faup = Faup()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
update_version = 'v2.7'
|
||||
|
||||
r_serv_db.set('ail:update_in_progress', update_version)
|
||||
r_serv_db.set('ail:current_background_update', update_version)
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'tags update')
|
||||
|
||||
update_db()
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
|
||||
|
||||
end = time.time()
|
||||
print('ALL domains tags updated in {} s'.format(end - start_deb))
|
||||
|
||||
r_serv_db.delete('ail:update_in_progress')
|
||||
r_serv_db.delete('ail:current_background_script')
|
||||
r_serv_db.delete('ail:current_background_script_stat')
|
||||
r_serv_db.delete('ail:current_background_update')
|
||||
r_serv_db.srem('ail:to_update', update_version)
|
|
@ -90,7 +90,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
|
|||
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
}
|
||||
|
||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||
|
|
Loading…
Reference in a new issue