mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [updates] remove v2.4 update + cleanup
This commit is contained in:
Terrtia
parent
4c2746a2d6
commit
e67252c65b
33 changed files with 314 additions and 503 deletions
|
|
@ -2,14 +2,15 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
|
||||
class AIL_Updater(object):
|
||||
"""docstring for AIL_Updater."""
|
||||
|
|
@ -38,9 +39,9 @@ class AIL_Updater(object):
|
|||
"""
|
||||
Update DB version
|
||||
"""
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
if self.f_version > self.current_f_version:
|
||||
self.r_serv.set('ail:version', self.version)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,7 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
|
|
@ -17,7 +14,7 @@ from lib import ail_updates
|
|||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='AIL default update')
|
||||
parser.add_argument('-t', help='version tag' , type=str, dest='tag', required=True)
|
||||
parser.add_argument('-t', help='version tag', type=str, dest='tag', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.tag:
|
||||
|
|
@ -28,7 +25,7 @@ if __name__ == '__main__':
|
|||
update_tag = args.tag.replace(' ', '')
|
||||
if not ail_updates.check_version(update_tag):
|
||||
parser.print_help()
|
||||
print(f'Error: Invalid update tag {update_tag})
|
||||
print(f'Error: Invalid update tag {update_tag}')
|
||||
sys.exit(0)
|
||||
|
||||
start_deb = time.time()
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
from lib import ConfigLoader
|
||||
|
||||
def update_tracked_terms(main_key, tracked_container_key):
|
||||
for tracked_item in r_serv_term.smembers(main_key):
|
||||
|
|
|
|||
|
|
@ -4,11 +4,10 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
|
|
|
|||
|
|
@ -4,13 +4,12 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
|
|
@ -26,6 +25,7 @@ def substract_date(date_from, date_to):
|
|||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import time
|
|||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def tags_key_fusion(old_item_path_key, new_item_path_key):
|
||||
print('fusion:')
|
||||
|
|
@ -17,10 +16,10 @@ def tags_key_fusion(old_item_path_key, new_item_path_key):
|
|||
r_serv_metadata.sadd(new_item_path_key, tag)
|
||||
r_serv_metadata.srem(old_item_path_key, tag)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
|
@ -38,7 +37,7 @@ if __name__ == '__main__':
|
|||
print('Updating ARDB_Tags ...')
|
||||
start = time.time()
|
||||
|
||||
#update item metadata tags
|
||||
# update item metadata tags
|
||||
tag_not_updated = True
|
||||
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
|
||||
nb_updated = 0
|
||||
|
|
@ -57,7 +56,7 @@ if __name__ == '__main__':
|
|||
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
|
||||
tag_not_updated = False
|
||||
else:
|
||||
progress = int((nb_updated * 100) /total_to_update)
|
||||
progress = int((nb_updated * 100) / total_to_update)
|
||||
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
|
|
@ -66,7 +65,6 @@ if __name__ == '__main__':
|
|||
|
||||
end = time.time()
|
||||
|
||||
|
||||
print('Updating ARDB_Tags Done: {} s'.format(end - start))
|
||||
|
||||
r_serv.sadd('ail:update_v1.5', 'tags_background')
|
||||
|
|
|
|||
|
|
@ -4,11 +4,10 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
|
|
|||
|
|
@ -4,11 +4,10 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
|
@ -19,8 +18,8 @@ if __name__ == '__main__':
|
|||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v1.7')
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -4,11 +4,10 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
|
@ -19,7 +18,7 @@ if __name__ == '__main__':
|
|||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v2.0')
|
||||
|
||||
# use new update_date format
|
||||
|
|
@ -30,5 +29,5 @@ if __name__ == '__main__':
|
|||
r_serv.hset('ail:update_date', tag, date_tag)
|
||||
r_serv.delete('ail:update_date_{}'.format(tag))
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -2,51 +2,58 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Item
|
||||
import Term
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from packages import Term
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
from lib import ConfigLoader
|
||||
|
||||
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
return new.join(li)
|
||||
|
||||
def get_item_id(full_path):
|
||||
return full_path.replace(PASTES_FOLDER, '', 1)
|
||||
|
||||
def get_item_date(id_item):
|
||||
l_dir = id_item.split('/')
|
||||
return f'{l_dir[-4]}{l_dir[-3]}{l_dir[-2]}'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending")
|
||||
r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq")
|
||||
config_loader = None
|
||||
|
||||
r_serv_term_stats.flushdb()
|
||||
|
||||
#convert all regex:
|
||||
# convert all regex:
|
||||
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
||||
for regex in all_regex:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
|
||||
|
||||
new_term = regex[1:-1]
|
||||
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
|
||||
'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
|
||||
# Invalid Tracker => remove it
|
||||
|
|
@ -58,16 +65,16 @@ if __name__ == '__main__':
|
|||
|
||||
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
||||
for token in all_tokens:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
|
||||
|
||||
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
|
|
@ -78,8 +85,8 @@ if __name__ == '__main__':
|
|||
|
||||
all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
||||
for curr_set in all_set:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
|
||||
|
||||
to_remove = ',{}'.format(curr_set.split(',')[-1])
|
||||
new_set = rreplace(curr_set, to_remove, '', 1)
|
||||
|
|
@ -91,8 +98,8 @@ if __name__ == '__main__':
|
|||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
|
|
@ -103,8 +110,8 @@ if __name__ == '__main__':
|
|||
|
||||
r_serv_termfreq.flushdb()
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v2.2')
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
new_version = 'v2.4'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
|
||||
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
|
||||
r_serv.set('update:nb_elem_converted',0)
|
||||
|
||||
# Add background update
|
||||
r_serv.sadd('ail:to_update', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
cp ${AIL_BIN}/packages/config.cfg ${AIL_HOME}/configs/core.cfg
|
||||
rm ${AIL_BIN}/packages/config.cfg
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.4/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Item
|
||||
import Tag
|
||||
from Cryptocurrency import cryptocurrency
|
||||
from Pgp import pgp
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Decoded
|
||||
import Domain
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
def update_domain_by_item(domain_obj, item_id):
|
||||
domain_name = domain_obj.get_domain_name()
|
||||
# update domain tags
|
||||
for tag in Tag.get_obj_tag(item_id):
|
||||
if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
||||
Tag.add_tag("domain", tag, domain_name, obj_date=Item.get_item_date(item_id))
|
||||
|
||||
# update domain correlation
|
||||
item_correlation = Item.get_item_all_correlation(item_id)
|
||||
|
||||
for correlation_name in item_correlation:
|
||||
for correlation_type in item_correlation[correlation_name]:
|
||||
if correlation_name in ('pgp', 'cryptocurrency'):
|
||||
for correl_value in item_correlation[correlation_name][correlation_type]:
|
||||
if correlation_name=='pgp':
|
||||
pgp.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||
if correlation_name=='cryptocurrency':
|
||||
cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||
if correlation_name=='decoded':
|
||||
for decoded_item in item_correlation['decoded']:
|
||||
Decoded.save_domain_decoded(domain_name, decoded_item)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv.set('ail:current_background_script', 'domain update')
|
||||
|
||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||
if not nb_elem_to_update:
|
||||
nb_elem_to_update = 0
|
||||
else:
|
||||
nb_elem_to_update = int(nb_elem_to_update)
|
||||
|
||||
while True:
|
||||
domain = r_serv_onion.spop('domain_update_v2.4')
|
||||
if domain is not None:
|
||||
print(domain)
|
||||
domain = Domain.Domain(domain)
|
||||
for domain_history in domain.get_domain_history():
|
||||
|
||||
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||
if "items" in domain_item:
|
||||
for item_dict in domain_item['items']:
|
||||
update_domain_by_item(domain, item_dict['id'])
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
else:
|
||||
r_serv.delete('update:nb_elem_to_convert')
|
||||
r_serv.delete('update:nb_elem_converted')
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
sys.exit(0)
|
||||
|
|
@ -2,14 +2,12 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.5'
|
||||
|
||||
|
|
@ -30,8 +28,8 @@ if __name__ == '__main__':
|
|||
r_serv.sadd('user_role:user_no_api', user)
|
||||
r_serv.sadd('user_role:read_only', user)
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -2,19 +2,16 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.6'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
|
@ -23,8 +20,8 @@ if __name__ == '__main__':
|
|||
|
||||
r_serv.sadd('ail:to_update', new_version)
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -4,21 +4,19 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Item
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
def get_domain(item_id):
|
||||
item_id = item_id.split('/')
|
||||
item_id = item_id[-1]
|
||||
return item_id[:-36]
|
||||
|
||||
def get_all_item(screenshot_sha256):
|
||||
return r_serv_onion.smembers('screenshot:{}'.format(screenshot_sha256))
|
||||
def get_all_item(s_sha256):
|
||||
return r_serv_onion.smembers(f'screenshot:{s_sha256}')
|
||||
|
||||
def sanitize_domain(domain):
|
||||
faup.decode(domain)
|
||||
|
|
@ -30,27 +28,24 @@ def sanitize_domain(domain):
|
|||
pass
|
||||
return domain_sanitized.lower()
|
||||
|
||||
def update_db(screenshot_sha256):
|
||||
screenshot_items = get_all_item(screenshot_sha256)
|
||||
def update_db(s_sha256):
|
||||
screenshot_items = get_all_item(s_sha256)
|
||||
if screenshot_items:
|
||||
for item_id in screenshot_items:
|
||||
item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path
|
||||
domain = Item.get_domain(item_id)
|
||||
item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path
|
||||
domain = get_domain(item_id)
|
||||
|
||||
domain_sanitized = sanitize_domain(domain)
|
||||
if domain != domain_sanitized:
|
||||
r_serv_onion.sadd('incorrect_domain', domain)
|
||||
domain = domain_sanitized
|
||||
|
||||
#print(item_id)
|
||||
#print(domain)
|
||||
|
||||
r_serv_onion.sadd('domain_screenshot:{}'.format(domain), screenshot_sha256)
|
||||
r_serv_onion.sadd('screenshot_domain:{}'.format(screenshot_sha256), domain)
|
||||
r_serv_onion.sadd('domain_screenshot:{}'.format(domain), s_sha256)
|
||||
r_serv_onion.sadd('screenshot_domain:{}'.format(s_sha256), domain)
|
||||
else:
|
||||
pass
|
||||
# broken screenshot
|
||||
r_serv_onion.sadd('broken_screenshot', screenshot_sha256)
|
||||
r_serv_onion.sadd('broken_screenshot', s_sha256)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
@ -77,15 +72,15 @@ if __name__ == '__main__':
|
|||
|
||||
if os.path.isdir(SCREENSHOT_FOLDER):
|
||||
for root, dirs, files in os.walk(SCREENSHOT_FOLDER, topdown=False):
|
||||
#print(dirs)
|
||||
# print(dirs)
|
||||
for name in files:
|
||||
nb = nb + 1
|
||||
screenshot_sha256 = os.path.join(root, name)
|
||||
screenshot_sha256 = screenshot_sha256[:-4] # remove .png
|
||||
screenshot_sha256 = screenshot_sha256[:-4] # remove .png
|
||||
screenshot_sha256 = screenshot_sha256.replace(SCREENSHOT_FOLDER, '', 1)
|
||||
screenshot_sha256 = screenshot_sha256.replace('/', '')
|
||||
update_db(screenshot_sha256)
|
||||
#print('Screenshot updated: {}'.format(nb))
|
||||
# print('Screenshot updated: {}'.format(nb))
|
||||
if nb % 1000 == 0:
|
||||
r_serv_db.set('ail:current_background_script', 'screenshot updated: {}'.format(nb))
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,12 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.7'
|
||||
|
||||
|
|
@ -32,8 +30,8 @@ if __name__ == '__main__':
|
|||
r_serv.set('ail:update_v2.7:deletetagrange', 1)
|
||||
#### ####
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -4,19 +4,13 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from packages import Date
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tag
|
||||
from lib import ConfigLoader
|
||||
|
||||
def sanitize_domain(domain):
|
||||
faup.decode(domain)
|
||||
|
|
@ -28,8 +22,23 @@ def sanitize_domain(domain):
|
|||
pass
|
||||
return domain_sanitized.lower()
|
||||
|
||||
def get_all_obj_tags(obj_type):
|
||||
return list(r_serv_tags.smembers(f'list_tags:{obj_type}'))
|
||||
|
||||
def add_global_tag(tag, object_type=None):
|
||||
r_serv_tags.sadd('list_tags', tag)
|
||||
if object_type:
|
||||
r_serv_tags.sadd('list_tags:{}'.format(object_type), tag)
|
||||
|
||||
def get_obj_tag(object_id):
|
||||
res = r_serv_metadata.smembers('tag:{}'.format(object_id))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def delete_domain_tag_daterange():
|
||||
all_domains_tags = Tag.get_all_obj_tags('domain')
|
||||
all_domains_tags = get_all_obj_tags('domain')
|
||||
nb_updated = 0
|
||||
nb_to_update = len(all_domains_tags)
|
||||
if nb_to_update == 0:
|
||||
|
|
@ -49,13 +58,13 @@ def update_domain_tags(domain):
|
|||
r_serv_onion.sadd('incorrect_domain', domain)
|
||||
domain = domain_sanitized
|
||||
|
||||
domain_tags = Tag.get_obj_tag(domain)
|
||||
domain_tags = get_obj_tag(domain)
|
||||
for tag in domain_tags:
|
||||
# delete incorrect tags
|
||||
if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"':
|
||||
r_serv_metadata.srem('tag:{}'.format(domain), tag)
|
||||
else:
|
||||
Tag.add_global_tag(tag, object_type='domain')
|
||||
add_global_tag(tag, object_type='domain')
|
||||
r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
|
||||
|
||||
def update_progress(refresh_time, nb_updated, nb_elem_to_update):
|
||||
|
|
@ -88,6 +97,7 @@ def update_db():
|
|||
r_serv_onion.sort('full_onion_up', alpha=True)
|
||||
r_serv_onion.sort('full_regular_up', alpha=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
|
@ -113,6 +123,5 @@ if __name__ == '__main__':
|
|||
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
|
||||
|
||||
end = time.time()
|
||||
print('ALL domains tags updated in {} s'.format(end - start_deb))
|
||||
|
|
|
|||
|
|
@ -3,27 +3,20 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
new_version = 'v3.0'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
updater = Updater('v3.0')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -3,37 +3,20 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
new_version = 'v3.1.1'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#### NEW EXPORTER
|
||||
# remove old tags errors
|
||||
#r_serv_db.delete('mess_not_saved_export')
|
||||
|
||||
# move solo tags to export in tags_db
|
||||
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
|
||||
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
|
||||
# # TODO: save them in tags db
|
||||
#### NEW EXPORTER
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
updater = Updater('v3.1.1')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -3,27 +3,21 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
new_version = 'v3.1'
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.1')
|
||||
updater.run_update()
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
|
|||
|
|
@ -3,37 +3,20 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
new_version = 'v3.2'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#### NEW EXPORTER
|
||||
# remove old tags errors
|
||||
#r_serv_db.delete('mess_not_saved_export')
|
||||
|
||||
# move solo tags to export in tags_db
|
||||
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
|
||||
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
|
||||
# # TODO: save them in tags db
|
||||
#### NEW EXPORTER
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
updater = Updater('v3.2')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -3,27 +3,20 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
new_version = 'v3.3'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
updater = Updater('v3.3')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -27,12 +23,12 @@ class Updater(AIL_Updater):
|
|||
"""
|
||||
self.r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up')
|
||||
self.r_serv.set('update:nb_elem_to_convert', self.r_serv_onion.scard('domain_update_v3.4'))
|
||||
self.r_serv.set('update:nb_elem_converted',0)
|
||||
self.r_serv.set('update:nb_elem_converted', 0)
|
||||
|
||||
# Add background update
|
||||
self.r_serv.sadd('ail:to_update', self.version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.4')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,33 +2,95 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Domain
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import Domain
|
||||
|
||||
def get_domain_type(domain_name):
|
||||
if str(domain_name).endswith('.onion'):
|
||||
return 'onion'
|
||||
else:
|
||||
return 'regular'
|
||||
|
||||
def add_domain_language(domain_name, language):
|
||||
language = language.split('-')[0]
|
||||
domain_type = get_domain_type(domain_name)
|
||||
r_serv_onion.sadd('all_domains_languages', language)
|
||||
r_serv_onion.sadd(f'all_domains_languages:{domain_type}', language)
|
||||
r_serv_onion.sadd(f'language:domains:{domain_type}:{language}', domain_name)
|
||||
r_serv_onion.sadd(f'domain:language:{domain_name}', language)
|
||||
|
||||
def add_domain_languages_by_item_id(domain_name, item_id):
|
||||
item = Item(item_id)
|
||||
for lang in item.get_languages():
|
||||
add_domain_language(domain_name, lang.language)
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
def update_domain_language(domain_obj, item_id):
|
||||
domain_name = domain_obj.get_domain_name()
|
||||
Domain.add_domain_languages_by_item_id(domain_name, item_id)
|
||||
add_domain_languages_by_item_id(domain_name, item_id)
|
||||
|
||||
def get_domain_history(domain_type, domain_name):
|
||||
return r_serv_onion.zrange(f'crawler_history_{domain_type}:{domain_name}:80', 0, -1, withscores=True)
|
||||
|
||||
|
||||
def get_item_children(item_id):
|
||||
return r_serv_metadata.smembers(f'paste_children:{item_id}')
|
||||
|
||||
def get_domain_items(domain_name, root_item_id):
|
||||
dom_item = get_domain_item_children(domain_name, root_item_id)
|
||||
dom_item.append(root_item_id)
|
||||
return dom_item
|
||||
|
||||
def is_item_in_domain(domain_name, item_id):
|
||||
is_in_domain = False
|
||||
domain_length = len(domain_name)
|
||||
if len(item_id) > (domain_length+48):
|
||||
if item_id[-36-domain_length:-36] == domain_name:
|
||||
is_in_domain = True
|
||||
return is_in_domain
|
||||
|
||||
def get_domain_item_children(domain_name, root_item_id):
|
||||
all_items = []
|
||||
for item_id in get_item_children(root_item_id):
|
||||
if is_item_in_domain(domain_name, item_id):
|
||||
all_items.append(item_id)
|
||||
all_items.extend(get_domain_item_children(domain_name, item_id))
|
||||
return all_items
|
||||
|
||||
def get_domain_crawled_item_root(domain_name, domain_type, epoch):
|
||||
res = r_serv_onion.zrevrangebyscore(f'crawler_history_{domain_type}:{domain_name}:80', int(epoch), int(epoch))
|
||||
return {"root_item": res[0], "epoch": int(epoch)}
|
||||
|
||||
def get_domain_items_crawled(domain_name, domain_type, epoch):
|
||||
item_crawled = []
|
||||
item_root = get_domain_crawled_item_root(domain_name, domain_type, epoch)
|
||||
if item_root:
|
||||
if item_root['root_item'] != str(item_root['epoch']) and item_root['root_item']:
|
||||
for item_id in get_domain_items(domain_name, item_root['root_item']):
|
||||
item_crawled.append(item_id)
|
||||
return item_crawled
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
r_serv_db.set('ail:current_background_script', 'domain languages update')
|
||||
|
|
@ -39,18 +101,19 @@ if __name__ == '__main__':
|
|||
else:
|
||||
nb_elem_to_update = int(nb_elem_to_update)
|
||||
|
||||
#Domain._delete_all_domains_languages()
|
||||
# _delete_all_domains_languages()
|
||||
|
||||
while True:
|
||||
domain = r_serv_onion.spop('domain_update_v3.4')
|
||||
if domain is not None:
|
||||
print(domain)
|
||||
domain = str(domain)
|
||||
domain_t = get_domain_type(domain)
|
||||
domain = Domain.Domain(domain)
|
||||
for domain_history in domain.get_domain_history():
|
||||
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||
if "items" in domain_item:
|
||||
for item_dict in domain_item['items']:
|
||||
update_domain_language(domain, item_dict['id'])
|
||||
for domain_history in get_domain_history(domain_t, domain):
|
||||
domain_items = get_domain_items_crawled(domain, domain_t, domain_history[1])
|
||||
for id_item in domain_items:
|
||||
update_domain_language(domain, id_item)
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
|
|||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.5')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
|
|||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.6')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,18 +2,14 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tracker
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
from lib import Tracker
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -38,7 +34,7 @@ class Updater(AIL_Updater):
|
|||
# Add background update
|
||||
self.r_serv.sadd('ail:to_update', self.version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.7')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,29 +2,31 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tracker
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib import Tracker
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tracker = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
Tracker.r_serv_tracker = r_serv_tracker
|
||||
|
||||
r_serv_db.set('ail:current_background_script', 'trackers update')
|
||||
|
||||
|
|
@ -43,7 +45,6 @@ if __name__ == '__main__':
|
|||
# MAP TRACKER - ITEM_ID
|
||||
Tracker.fix_tracker_item_link(tracker_uuid)
|
||||
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
|
|||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.0')
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,14 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -21,17 +18,14 @@ class Updater(AIL_Updater):
|
|||
super(Updater, self).__init__(version)
|
||||
|
||||
def update(self):
|
||||
r_tracking = redis.StrictRedis( host='localhost',
|
||||
port=6382,
|
||||
db=2,
|
||||
decode_responses=True )
|
||||
|
||||
r_tracking = redis.StrictRedis(host='localhost',
|
||||
port=6382,
|
||||
db=2,
|
||||
decode_responses=True)
|
||||
# FLUSH OLD DB
|
||||
r_tracking.flushdb()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
updater = Updater('v4.1')
|
||||
|
||||
|
||||
updater.run_update()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from old_ail_updater import AIL_Updater
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
|
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
|
|||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.2')
|
||||
updater.run_update()
|
||||
|
|
|
|||
Loading…
Reference in a new issue