mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [update] remove old updates < 5.0
This commit is contained in:
Terrtia
parent
73bfe614df
commit
ff2b620647
52 changed files with 0 additions and 2636 deletions
|
|
@ -1,18 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Killing all screens ..."
|
||||
bash -c "bash ../../bin/LAUNCH.sh -k"
|
||||
echo ""
|
||||
echo "Updating ARDB ..."
|
||||
pushd ../../
|
||||
rm -r ardb
|
||||
pushd ardb/
|
||||
git clone https://github.com/yinqiwen/ardb.git
|
||||
git checkout 0.10 || exit 1
|
||||
make || exit 1
|
||||
popd
|
||||
popd
|
||||
echo "ARDB Updated"
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
|
||||
class AIL_Updater(object):
|
||||
"""docstring for AIL_Updater."""
|
||||
|
||||
def __init__(self, new_version):
|
||||
self.version = new_version
|
||||
self.start_time = time.time()
|
||||
|
||||
self.config = ConfigLoader.ConfigLoader()
|
||||
self.r_serv = self.config.get_redis_conn("ARDB_DB")
|
||||
|
||||
self.f_version = float(self.version[1:])
|
||||
self.current_f_version = self.r_serv.get('ail:version')
|
||||
if self.current_f_version:
|
||||
self.current_f_version = float(self.current_f_version[1:])
|
||||
else:
|
||||
self.current_f_version = 0
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
AIL DB update
|
||||
"""
|
||||
pass
|
||||
|
||||
def end_update(self):
|
||||
"""
|
||||
Update DB version
|
||||
"""
|
||||
# Set current ail version
|
||||
self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
# Set current ail version
|
||||
if self.f_version > self.current_f_version:
|
||||
self.r_serv.set('ail:version', self.version)
|
||||
|
||||
def run_update(self):
|
||||
self.update()
|
||||
self.end_update()
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
YELLOW="\\033[1;33m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $YELLOW"\t"
|
||||
echo -e "* ------------------------------------------------------------------"
|
||||
echo -e "\t"
|
||||
echo -e " - - - - - - - - PLEASE RELAUNCH AIL - - - - - - - - "
|
||||
echo -e "\t"
|
||||
echo -e "* ------------------------------------------------------------------"
|
||||
echo -e "\t"
|
||||
echo -e "\t"$DEFAULT
|
||||
|
||||
# fix invalid Updater version (kill parent):
|
||||
kill -SIGUSR1 `ps --pid $$ -oppid=`; exit
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
from lib import ConfigLoader
|
||||
|
||||
def update_tracked_terms(main_key, tracked_container_key):
|
||||
for tracked_item in r_serv_term.smembers(main_key):
|
||||
all_items = r_serv_term.smembers(tracked_container_key.format(tracked_item))
|
||||
for item_path in all_items:
|
||||
if PASTES_FOLDER in item_path:
|
||||
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||
r_serv_term.sadd(tracked_container_key.format(tracked_item), new_item_path)
|
||||
r_serv_term.srem(tracked_container_key.format(tracked_item), item_path)
|
||||
|
||||
def update_hash_item(has_type):
|
||||
#get all hash items:
|
||||
all_hash_items = r_serv_tag.smembers('infoleak:automatic-detection=\"{}\"'.format(has_type))
|
||||
for item_path in all_hash_items:
|
||||
if PASTES_FOLDER in item_path:
|
||||
base64_key = '{}_paste:{}'.format(has_type, item_path)
|
||||
hash_key = 'hash_paste:{}'.format(item_path)
|
||||
|
||||
if r_serv_metadata.exists(base64_key):
|
||||
new_base64_key = base64_key.replace(PASTES_FOLDER, '', 1)
|
||||
res = r_serv_metadata.renamenx(base64_key, new_base64_key)
|
||||
if res == 0:
|
||||
print('same key, double name: {}'.format(item_path))
|
||||
# fusion
|
||||
all_key = r_serv_metadata.smembers(base64_key)
|
||||
for elem in all_key:
|
||||
r_serv_metadata.sadd(new_base64_key, elem)
|
||||
r_serv_metadata.srem(base64_key, elem)
|
||||
|
||||
if r_serv_metadata.exists(hash_key):
|
||||
new_hash_key = hash_key.replace(PASTES_FOLDER, '', 1)
|
||||
res = r_serv_metadata.renamenx(hash_key, new_hash_key)
|
||||
if res == 0:
|
||||
print('same key, double name: {}'.format(item_path))
|
||||
# fusion
|
||||
all_key = r_serv_metadata.smembers(hash_key)
|
||||
for elem in all_key:
|
||||
r_serv_metadata.sadd(new_hash_key, elem)
|
||||
r_serv_metadata.srem(hash_key, elem)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_tag = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_term = config_loader.get_redis_conn("ARDB_TermFreq")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv.set('ail:current_background_script', 'metadata')
|
||||
|
||||
## Update metadata ##
|
||||
print('Updating ARDB_Metadata ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 0)
|
||||
|
||||
# Update base64
|
||||
update_hash_item('base64')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 20)
|
||||
# Update binary
|
||||
update_hash_item('binary')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 40)
|
||||
# Update binary
|
||||
update_hash_item('hexadecimal')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 60)
|
||||
|
||||
total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"')
|
||||
nb_updated = 0
|
||||
last_progress = 0
|
||||
|
||||
# Update onion metadata
|
||||
all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
|
||||
for item_path in all_crawled_items:
|
||||
domain = None
|
||||
if PASTES_FOLDER in item_path:
|
||||
old_item_metadata = 'paste_metadata:{}'.format(item_path)
|
||||
item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||
new_item_metadata = 'paste_metadata:{}'.format(item_path)
|
||||
res = r_serv_metadata.renamenx(old_item_metadata, new_item_metadata)
|
||||
#key already exist
|
||||
if res == 0:
|
||||
r_serv_metadata.delete(old_item_metadata)
|
||||
|
||||
# update domain port
|
||||
domain = r_serv_metadata.hget(new_item_metadata, 'domain')
|
||||
if domain:
|
||||
if domain[-3:] != ':80':
|
||||
r_serv_metadata.hset(new_item_metadata, 'domain', '{}:80'.format(domain))
|
||||
super_father = r_serv_metadata.hget(new_item_metadata, 'super_father')
|
||||
if super_father:
|
||||
if PASTES_FOLDER in super_father:
|
||||
r_serv_metadata.hset(new_item_metadata, 'super_father', super_father.replace(PASTES_FOLDER, '', 1))
|
||||
father = r_serv_metadata.hget(new_item_metadata, 'father')
|
||||
if father:
|
||||
if PASTES_FOLDER in father:
|
||||
r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
|
||||
|
||||
nb_updated += 1
|
||||
progress = int((nb_updated * 30) /total_onion)
|
||||
print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60))
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
r_serv.set('ail:current_background_script_stat', progress + 60)
|
||||
last_progress = progress
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 90)
|
||||
|
||||
## update tracked term/set/regex
|
||||
# update tracked term
|
||||
update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 93)
|
||||
# update tracked set
|
||||
update_tracked_terms('TrackedSetSet', 'set_{}')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 96)
|
||||
# update tracked regex
|
||||
update_tracked_terms('TrackedRegexSet', 'regex_{}')
|
||||
|
||||
#update stats
|
||||
r_serv.set('ail:current_background_script_stat', 100)
|
||||
##
|
||||
|
||||
end = time.time()
|
||||
|
||||
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
|
||||
print()
|
||||
|
||||
r_serv.sadd('ail:update_v1.5', 'metadata')
|
||||
|
||||
##
|
||||
#Key, Dynamic Update
|
||||
##
|
||||
#paste_children
|
||||
#nb_seen_hash, base64_hash, binary_hash
|
||||
#paste_onion_external_links
|
||||
#misp_events, hive_cases
|
||||
##
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
def get_date_epoch(date):
|
||||
return int(datetime.datetime(int(date[0:4]), int(date[4:6]), int(date[6:8])).timestamp())
|
||||
|
||||
def get_domain_root_from_paste_childrens(item_father, domain):
|
||||
item_children = r_serv_metadata.smembers('paste_children:{}'.format(item_father))
|
||||
domain_root = ''
|
||||
for item_path in item_children:
|
||||
# remove absolute_path
|
||||
if PASTES_FOLDER in item_path:
|
||||
r_serv_metadata.srem('paste_children:{}'.format(item_father), item_path)
|
||||
item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||
r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_path)
|
||||
if domain in item_path:
|
||||
domain_root = item_path
|
||||
return domain_root
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_tag = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv.set('ail:current_background_script', 'onions')
|
||||
r_serv.set('ail:current_background_script_stat', 0)
|
||||
|
||||
## Update Onion ##
|
||||
print('Updating ARDB_Onion ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
|
||||
# clean down domain from db
|
||||
date_from = '20180929'
|
||||
date_today = datetime.date.today().strftime("%Y%m%d")
|
||||
for date in substract_date(date_from, date_today):
|
||||
|
||||
onion_down = r_serv_onion.smembers('onion_down:{}'.format(date))
|
||||
#print(onion_down)
|
||||
for onion_domain in onion_down:
|
||||
if not r_serv_onion.sismember('full_onion_up', onion_domain):
|
||||
# delete history
|
||||
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
|
||||
if all_onion_history:
|
||||
for date_history in all_onion_history:
|
||||
#print('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||
r_serv_onion.delete('onion_history:{}'.format(onion_domain))
|
||||
|
||||
#stats
|
||||
total_domain = r_serv_onion.scard('full_onion_up')
|
||||
nb_updated = 0
|
||||
last_progress = 0
|
||||
|
||||
# clean up domain
|
||||
all_domain_up = r_serv_onion.smembers('full_onion_up')
|
||||
for onion_domain in all_domain_up:
|
||||
# delete history
|
||||
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
|
||||
if all_onion_history:
|
||||
for date_history in all_onion_history:
|
||||
print('--------')
|
||||
print('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||
item_father = r_serv_onion.lrange('onion_history:{}:{}'.format(onion_domain, date_history), 0, 0)
|
||||
print('item_father: {}'.format(item_father))
|
||||
try:
|
||||
item_father = item_father[0]
|
||||
except IndexError:
|
||||
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||
continue
|
||||
#print(item_father)
|
||||
# delete old history
|
||||
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
|
||||
# create new history
|
||||
root_key = get_domain_root_from_paste_childrens(item_father, onion_domain)
|
||||
if root_key:
|
||||
r_serv_onion.zadd(f'crawler_history_onion:{onion_domain}:80', {root_key: get_date_epoch(date_history)})
|
||||
print('crawler_history_onion:{}:80 {} {}'.format(onion_domain, get_date_epoch(date_history), root_key))
|
||||
#update service metadata: paste_parent
|
||||
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'paste_parent', root_key)
|
||||
|
||||
r_serv_onion.delete('onion_history:{}'.format(onion_domain))
|
||||
|
||||
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
|
||||
r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
|
||||
|
||||
nb_updated += 1
|
||||
progress = int((nb_updated * 100) /total_domain)
|
||||
print('{}/{} updated {}%'.format(nb_updated, total_domain, progress))
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
r_serv.set('ail:current_background_script_stat', progress)
|
||||
last_progress = progress
|
||||
|
||||
|
||||
end = time.time()
|
||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||
print()
|
||||
print('Done in {} s'.format(end - start_deb))
|
||||
|
||||
r_serv.sadd('ail:update_v1.5', 'onions')
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
return new.join(li)
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"))
|
||||
NEW_SCREENSHOT_FOLDER = config_loader.get_files_directory('screenshot')
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_tag = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv.set('ail:current_background_script', 'crawled_screenshot')
|
||||
r_serv.set('ail:current_background_script_stat', 0)
|
||||
|
||||
## Update Onion ##
|
||||
print('Updating ARDB_Onion ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
|
||||
# clean down domain from db
|
||||
date_from = '20180801'
|
||||
date_today = datetime.date.today().strftime("%Y%m%d")
|
||||
list_date = substract_date(date_from, date_today)
|
||||
nb_done = 0
|
||||
last_progress = 0
|
||||
total_to_update = len(list_date)
|
||||
for date in list_date:
|
||||
screenshot_dir = os.path.join(SCREENSHOT_FOLDER, date[0:4], date[4:6], date[6:8])
|
||||
if os.path.isdir(screenshot_dir):
|
||||
print(screenshot_dir)
|
||||
for file in os.listdir(screenshot_dir):
|
||||
if file.endswith(".png"):
|
||||
index += 1
|
||||
#print(file)
|
||||
|
||||
img_path = os.path.join(screenshot_dir, file)
|
||||
with open(img_path, 'br') as f:
|
||||
image_content = f.read()
|
||||
|
||||
hash = sha256(image_content).hexdigest()
|
||||
img_dir_path = os.path.join(hash[0:2], hash[2:4], hash[4:6], hash[6:8], hash[8:10], hash[10:12])
|
||||
filename_img = os.path.join(NEW_SCREENSHOT_FOLDER, img_dir_path, hash[12:] +'.png')
|
||||
dirname = os.path.dirname(filename_img)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
if not os.path.exists(filename_img):
|
||||
os.rename(img_path, filename_img)
|
||||
else:
|
||||
os.remove(img_path)
|
||||
|
||||
item = os.path.join('crawled', date[0:4], date[4:6], date[6:8], file[:-4])
|
||||
# add item metadata
|
||||
r_serv_metadata.hset('paste_metadata:{}'.format(item), 'screenshot', hash)
|
||||
# add sha256 metadata
|
||||
r_serv_onion.sadd('screenshot:{}'.format(hash), item)
|
||||
|
||||
if file.endswith('.pnghar.txt'):
|
||||
har_path = os.path.join(screenshot_dir, file)
|
||||
new_file = rreplace(file, '.pnghar.txt', '.json', 1)
|
||||
new_har_path = os.path.join(screenshot_dir, new_file)
|
||||
os.rename(har_path, new_har_path)
|
||||
|
||||
progress = int((nb_done * 100) /total_to_update)
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
r_serv.set('ail:current_background_script_stat', progress)
|
||||
print('{}/{} screenshot updated {}%'.format(nb_done, total_to_update, progress))
|
||||
last_progress = progress
|
||||
|
||||
nb_done += 1
|
||||
|
||||
r_serv.set('ail:current_background_script_stat', 100)
|
||||
|
||||
|
||||
end = time.time()
|
||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||
print()
|
||||
print('Done in {} s'.format(end - start_deb))
|
||||
|
||||
r_serv.set('ail:current_background_script_stat', 100)
|
||||
r_serv.sadd('ail:update_v1.5', 'crawled_screenshot')
|
||||
if r_serv.scard('ail:update_v1.5') != 5:
|
||||
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_tag = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
r_important_paste_2018 = redis.StrictRedis(
|
||||
host=config_loader.get_config_str("ARDB_Metadata", "host"),
|
||||
port=config_loader.get_config_int("ARDB_Metadata", "port"),
|
||||
db=2018,
|
||||
decode_responses=True)
|
||||
|
||||
r_important_paste_2019 = redis.StrictRedis(
|
||||
host=config_loader.get_config_str("ARDB_Metadata", "host"),
|
||||
port=config_loader.get_config_int("ARDB_Metadata", "port"),
|
||||
db=2019,
|
||||
decode_responses=True)
|
||||
|
||||
config_loader = None
|
||||
|
||||
r_serv.set('ail:current_background_script', 'tags')
|
||||
r_serv.set('ail:current_background_script_stat', 0)
|
||||
|
||||
if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'):
|
||||
|
||||
print('Updating ARDB_Tags ...')
|
||||
index = 0
|
||||
nb_tags_to_update = 0
|
||||
nb_updated = 0
|
||||
last_progress = 0
|
||||
start = time.time()
|
||||
|
||||
tags_list = r_serv_tag.smembers('list_tags')
|
||||
# create temp tags metadata
|
||||
tag_metadata = {}
|
||||
for tag in tags_list:
|
||||
tag_metadata[tag] = {}
|
||||
tag_metadata[tag]['first_seen'] = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
||||
if tag_metadata[tag]['first_seen'] is None:
|
||||
tag_metadata[tag]['first_seen'] = 99999999
|
||||
else:
|
||||
tag_metadata[tag]['first_seen'] = int(tag_metadata[tag]['first_seen'])
|
||||
|
||||
tag_metadata[tag]['last_seen'] = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
if tag_metadata[tag]['last_seen'] is None:
|
||||
tag_metadata[tag]['last_seen'] = 0
|
||||
else:
|
||||
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
|
||||
nb_tags_to_update += r_serv_tag.scard(tag)
|
||||
|
||||
if nb_tags_to_update == 0:
|
||||
nb_tags_to_update = 1
|
||||
|
||||
for tag in tags_list:
|
||||
|
||||
all_item = r_serv_tag.smembers(tag)
|
||||
for item_path in all_item:
|
||||
splitted_item_path = item_path.split('/')
|
||||
#print(tag)
|
||||
#print(item_path)
|
||||
try:
|
||||
item_date = int( ''.join([splitted_item_path[-4], splitted_item_path[-3], splitted_item_path[-2]]) )
|
||||
except IndexError:
|
||||
r_serv_tag.srem(tag, item_path)
|
||||
continue
|
||||
|
||||
# remove absolute path
|
||||
new_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||
if new_path != item_path:
|
||||
# save in queue absolute path to remove
|
||||
r_serv_tag.sadd('maj:v1.5:absolute_path_to_rename', item_path)
|
||||
|
||||
# update metadata first_seen
|
||||
if item_date < tag_metadata[tag]['first_seen']:
|
||||
tag_metadata[tag]['first_seen'] = item_date
|
||||
r_serv_tag.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
|
||||
|
||||
# update metadata last_seen
|
||||
if item_date > tag_metadata[tag]['last_seen']:
|
||||
tag_metadata[tag]['last_seen'] = item_date
|
||||
last_seen_db = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
if last_seen_db:
|
||||
if item_date > int(last_seen_db):
|
||||
r_serv_tag.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
|
||||
else:
|
||||
tag_metadata[tag]['last_seen'] = last_seen_db
|
||||
|
||||
r_serv_tag.sadd('{}:{}'.format(tag, item_date), new_path)
|
||||
r_serv_tag.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
||||
|
||||
# clean db
|
||||
r_serv_tag.srem(tag, item_path)
|
||||
index = index + 1
|
||||
|
||||
nb_updated += 1
|
||||
progress = int((nb_updated * 100) /nb_tags_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress))
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
r_serv.set('ail:current_background_script_stat', progress)
|
||||
last_progress = progress
|
||||
|
||||
#flush browse importante pastes db
|
||||
try:
|
||||
r_important_paste_2018.flushdb()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
r_important_paste_2019.flushdb()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
end = time.time()
|
||||
|
||||
print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start))
|
||||
|
||||
r_serv.sadd('ail:update_v1.5', 'tags')
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def tags_key_fusion(old_item_path_key, new_item_path_key):
|
||||
print('fusion:')
|
||||
print(old_item_path_key)
|
||||
print(new_item_path_key)
|
||||
for tag in r_serv_metadata.smembers(old_item_path_key):
|
||||
r_serv_metadata.sadd(new_item_path_key, tag)
|
||||
r_serv_metadata.srem(old_item_path_key, tag)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_tag = config_loader.get_redis_conn("ARDB_Tags")
|
||||
config_loader = None
|
||||
|
||||
if r_serv.sismember('ail:update_v1.5', 'tags'):
|
||||
|
||||
r_serv.set('ail:current_background_script', 'tags_background')
|
||||
r_serv.set('ail:current_background_script_stat', 0)
|
||||
|
||||
print('Updating ARDB_Tags ...')
|
||||
start = time.time()
|
||||
|
||||
# update item metadata tags
|
||||
tag_not_updated = True
|
||||
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
|
||||
nb_updated = 0
|
||||
last_progress = 0
|
||||
if total_to_update > 0:
|
||||
while tag_not_updated:
|
||||
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
|
||||
old_tag_item_key = 'tag:{}'.format(item_path)
|
||||
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
|
||||
new_tag_item_key = 'tag:{}'.format(new_item_path)
|
||||
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
|
||||
if res == 0:
|
||||
tags_key_fusion(old_tag_item_key, new_tag_item_key)
|
||||
nb_updated += 1
|
||||
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
|
||||
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
|
||||
tag_not_updated = False
|
||||
else:
|
||||
progress = int((nb_updated * 100) / total_to_update)
|
||||
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
|
||||
# update progress stats
|
||||
if progress != last_progress:
|
||||
r_serv.set('ail:current_background_script_stat', progress)
|
||||
last_progress = progress
|
||||
|
||||
end = time.time()
|
||||
|
||||
print('Updating ARDB_Tags Done: {} s'.format(end - start))
|
||||
|
||||
r_serv.sadd('ail:update_v1.5', 'tags_background')
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
print()
|
||||
print('Updating ARDB_Onion ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
|
||||
# update crawler queue
|
||||
for elem in r_serv_onion.smembers('onion_crawler_queue'):
|
||||
if PASTES_FOLDER in elem:
|
||||
r_serv_onion.srem('onion_crawler_queue', elem)
|
||||
r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1))
|
||||
index = index +1
|
||||
for elem in r_serv_onion.smembers('onion_crawler_priority_queue'):
|
||||
if PASTES_FOLDER in elem:
|
||||
r_serv_onion.srem('onion_crawler_queue', elem)
|
||||
r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1))
|
||||
index = index +1
|
||||
|
||||
end = time.time()
|
||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||
print()
|
||||
|
||||
# Add background update
|
||||
r_serv.sadd('ail:to_update', 'v1.5')
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', 'v1.5')
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
||||
print('Done in {} s'.format(end - start_deb))
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh"
|
||||
#bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh"
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update DomainClassifier"$DEFAULT
|
||||
echo ""
|
||||
pip3 install --upgrade --force-reinstall git+https://github.com/D4-project/BGP-Ranking.git/@28013297efb039d2ebbce96ee2d89493f6ae56b0#subdirectory=client&egg=pybgpranking
|
||||
pip3 install --upgrade --force-reinstall git+https://github.com/adulau/DomainClassifier.git
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update Web thirdparty"$DEFAULT
|
||||
echo ""
|
||||
bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh &)"
|
||||
wait
|
||||
echo ""
|
||||
|
||||
bash ${AIL_BIN}LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Fixing ARDB ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v1.5/Update.py &
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v1.7')
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update DomainClassifier"$DEFAULT
|
||||
echo ""
|
||||
|
||||
cd $AIL_HOME
|
||||
git clone https://github.com/kazu-yamamoto/pgpdump.git
|
||||
cd pgpdump
|
||||
./configure
|
||||
make
|
||||
sudo make install
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update requirement"$DEFAULT
|
||||
echo ""
|
||||
pip3 install beautifulsoup4
|
||||
|
||||
bash ${AIL_BIN}LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v1.7/Update.py &
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t &
|
||||
wait
|
||||
|
||||
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v2.0')
|
||||
|
||||
# use new update_date format
|
||||
date_tag_to_replace = ['v1.5', 'v1.7']
|
||||
for tag in date_tag_to_replace:
|
||||
if r_serv.exists('ail:update_date_{}'.format(tag)):
|
||||
date_tag = r_serv.get('ail:update_date_{}'.format(tag))
|
||||
r_serv.hset('ail:update_date', tag, date_tag)
|
||||
r_serv.delete('ail:update_date_{}'.format(tag))
|
||||
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Create Self-Signed Certificate"$DEFAULT
|
||||
echo ""
|
||||
pushd ${AIL_BIN}/helper/gen_cert
|
||||
bash gen_root.sh
|
||||
wait
|
||||
bash gen_cert.sh
|
||||
wait
|
||||
popd
|
||||
|
||||
cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt
|
||||
cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update requirement"$DEFAULT
|
||||
echo ""
|
||||
pip3 install flask-login
|
||||
wait
|
||||
echo ""
|
||||
pip3 install bcrypt
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.0/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Create Default User"$DEFAULT
|
||||
echo ""
|
||||
python3 ${AIL_FLASK}create_default_user.py
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from packages import Term
|
||||
|
||||
from lib import ConfigLoader
|
||||
|
||||
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
return new.join(li)
|
||||
|
||||
def get_item_id(full_path):
|
||||
return full_path.replace(PASTES_FOLDER, '', 1)
|
||||
|
||||
def get_item_date(id_item):
|
||||
l_dir = id_item.split('/')
|
||||
return f'{l_dir[-4]}{l_dir[-3]}{l_dir[-2]}'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
|
||||
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending")
|
||||
r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq")
|
||||
config_loader = None
|
||||
|
||||
r_serv_term_stats.flushdb()
|
||||
|
||||
# Disabled. Checkout the v2.2 branch if you need it
|
||||
# # convert all regex:
|
||||
# all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
||||
# for regex in all_regex:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
|
||||
#
|
||||
# new_term = regex[1:-1]
|
||||
# res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
|
||||
# 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
#
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Regex Removed: {}'.format(regex))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedRegexSet', regex)
|
||||
#
|
||||
# all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
||||
# for token in all_tokens:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
|
||||
#
|
||||
# res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Token Removed: {}'.format(token))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedSetTermSet', token)
|
||||
#
|
||||
# all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
||||
# for curr_set in all_set:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
|
||||
#
|
||||
# to_remove = ',{}'.format(curr_set.split(',')[-1])
|
||||
# new_set = rreplace(curr_set, to_remove, '', 1)
|
||||
# new_set = new_set[2:]
|
||||
# new_set = new_set.replace(',', '')
|
||||
#
|
||||
# res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Set Removed: {}'.format(curr_set))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedSetSet', curr_set)
|
||||
|
||||
r_serv_termfreq.flushdb()
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', 'v2.2')
|
||||
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.2/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.5'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
r_serv.zadd('ail:all_role', {'user': 3})
|
||||
r_serv.zadd('ail:all_role', {'user_no_api': 4})
|
||||
r_serv.zadd('ail:all_role', {'read_only': 5})
|
||||
|
||||
for user in r_serv.hkeys('user:all'):
|
||||
r_serv.sadd('user_role:user', user)
|
||||
r_serv.sadd('user_role:user_no_api', user)
|
||||
r_serv.sadd('user_role:read_only', user)
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.5/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.6'
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
r_serv.sadd('ail:to_update', new_version)
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.6/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
def get_domain(item_id):
|
||||
item_id = item_id.split('/')
|
||||
item_id = item_id[-1]
|
||||
return item_id[:-36]
|
||||
|
||||
def get_all_item(s_sha256):
|
||||
return r_serv_onion.smembers(f'screenshot:{s_sha256}')
|
||||
|
||||
def sanitize_domain(domain):
|
||||
faup.decode(domain)
|
||||
domain_sanitized = faup.get()
|
||||
domain_sanitized = domain_sanitized['domain']
|
||||
try:
|
||||
domain_sanitized = domain_sanitized.decode()
|
||||
except:
|
||||
pass
|
||||
return domain_sanitized.lower()
|
||||
|
||||
def update_db(s_sha256):
|
||||
screenshot_items = get_all_item(s_sha256)
|
||||
if screenshot_items:
|
||||
for item_id in screenshot_items:
|
||||
item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path
|
||||
domain = get_domain(item_id)
|
||||
|
||||
domain_sanitized = sanitize_domain(domain)
|
||||
if domain != domain_sanitized:
|
||||
r_serv_onion.sadd('incorrect_domain', domain)
|
||||
domain = domain_sanitized
|
||||
|
||||
r_serv_onion.sadd('domain_screenshot:{}'.format(domain), s_sha256)
|
||||
r_serv_onion.sadd('screenshot_domain:{}'.format(s_sha256), domain)
|
||||
else:
|
||||
pass
|
||||
# broken screenshot
|
||||
r_serv_onion.sadd('broken_screenshot', s_sha256)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
faup = Faup()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes"))
|
||||
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot')
|
||||
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv_db.set('ail:update_in_progress', 'v2.6')
|
||||
r_serv_db.set('ail:current_background_update', 'v2.6')
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 20)
|
||||
r_serv_db.set('ail:current_background_script', 'screenshot update')
|
||||
|
||||
nb = 0
|
||||
|
||||
if os.path.isdir(SCREENSHOT_FOLDER):
|
||||
for root, dirs, files in os.walk(SCREENSHOT_FOLDER, topdown=False):
|
||||
# print(dirs)
|
||||
for name in files:
|
||||
nb = nb + 1
|
||||
screenshot_sha256 = os.path.join(root, name)
|
||||
screenshot_sha256 = screenshot_sha256[:-4] # remove .png
|
||||
screenshot_sha256 = screenshot_sha256.replace(SCREENSHOT_FOLDER, '', 1)
|
||||
screenshot_sha256 = screenshot_sha256.replace('/', '')
|
||||
update_db(screenshot_sha256)
|
||||
# print('Screenshot updated: {}'.format(nb))
|
||||
if nb % 1000 == 0:
|
||||
r_serv_db.set('ail:current_background_script', 'screenshot updated: {}'.format(nb))
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
|
||||
end = time.time()
|
||||
print('ALL screenshot updated: {} in {} s'.format(nb, end - start_deb))
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
new_version = 'v2.7'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv.sadd('ail:to_update', new_version)
|
||||
|
||||
#### Update tags ####
|
||||
r_serv_tags.sunionstore('list_tags:item', 'list_tags', [])
|
||||
r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', [])
|
||||
r_serv_onion.delete('incorrect_domain')
|
||||
r_serv.set('ail:update_v2.7:deletetagrange', 1)
|
||||
#### ####
|
||||
|
||||
# Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
# Set current ail version
|
||||
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.7/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from packages import Date
|
||||
|
||||
from lib import ConfigLoader
|
||||
|
||||
def sanitize_domain(domain):
|
||||
faup.decode(domain)
|
||||
domain_sanitized = faup.get()
|
||||
domain_sanitized = domain_sanitized['domain']
|
||||
try:
|
||||
domain_sanitized = domain_sanitized.decode()
|
||||
except:
|
||||
pass
|
||||
return domain_sanitized.lower()
|
||||
|
||||
def get_all_obj_tags(obj_type):
|
||||
return list(r_serv_tags.smembers(f'list_tags:{obj_type}'))
|
||||
|
||||
def add_global_tag(tag, object_type=None):
|
||||
r_serv_tags.sadd('list_tags', tag)
|
||||
if object_type:
|
||||
r_serv_tags.sadd('list_tags:{}'.format(object_type), tag)
|
||||
|
||||
def get_obj_tag(object_id):
|
||||
res = r_serv_metadata.smembers('tag:{}'.format(object_id))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def delete_domain_tag_daterange():
|
||||
all_domains_tags = get_all_obj_tags('domain')
|
||||
nb_updated = 0
|
||||
nb_to_update = len(all_domains_tags)
|
||||
if nb_to_update == 0:
|
||||
nb_to_update = 1
|
||||
refresh_time = time.time()
|
||||
l_dates = Date.substract_date('20191008', Date.get_today_date_str())
|
||||
for tag in all_domains_tags:
|
||||
for date_day in l_dates:
|
||||
r_serv_tags.delete('domain:{}:{}'.format(tag, date_day))
|
||||
nb_updated += 1
|
||||
refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
|
||||
r_serv_db.delete('ail:update_v2.7:deletetagrange')
|
||||
|
||||
def update_domain_tags(domain):
|
||||
domain_sanitized = sanitize_domain(domain)
|
||||
if domain != domain_sanitized:
|
||||
r_serv_onion.sadd('incorrect_domain', domain)
|
||||
domain = domain_sanitized
|
||||
|
||||
domain_tags = get_obj_tag(domain)
|
||||
for tag in domain_tags:
|
||||
# delete incorrect tags
|
||||
if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"':
|
||||
r_serv_metadata.srem('tag:{}'.format(domain), tag)
|
||||
else:
|
||||
add_global_tag(tag, object_type='domain')
|
||||
r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
|
||||
|
||||
def update_progress(refresh_time, nb_updated, nb_elem_to_update):
|
||||
if time.time() - refresh_time > 10:
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
refresh_time = time.time()
|
||||
|
||||
return refresh_time
|
||||
|
||||
def update_db():
|
||||
nb_updated = 0
|
||||
nb_to_update = r_serv_onion.scard('domain_update_v2.7')
|
||||
refresh_time = time.time()
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'domain tags update')
|
||||
domain = r_serv_onion.spop('domain_update_v2.7')
|
||||
while domain is not None:
|
||||
update_domain_tags(domain)
|
||||
nb_updated += 1
|
||||
refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
|
||||
domain = r_serv_onion.spop('domain_update_v2.7')
|
||||
if r_serv_db.exists('ail:update_v2.7:deletetagrange'):
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys')
|
||||
delete_domain_tag_daterange()
|
||||
|
||||
# sort all crawled domain
|
||||
r_serv_onion.sort('full_onion_up', alpha=True)
|
||||
r_serv_onion.sort('full_regular_up', alpha=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
faup = Faup()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
update_version = 'v2.7'
|
||||
|
||||
r_serv_db.set('ail:update_in_progress', update_version)
|
||||
r_serv_db.set('ail:current_background_update', update_version)
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 0)
|
||||
r_serv_db.set('ail:current_background_script', 'tags update')
|
||||
|
||||
update_db()
|
||||
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
|
||||
end = time.time()
|
||||
print('ALL domains tags updated in {} s'.format(end - start_deb))
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.0')
|
||||
updater.run_update()
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.0/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.1.1')
|
||||
updater.run_update()
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo -e $GREEN"Installing YARA ..."$DEFAULT
|
||||
pip3 install yara-python
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.1.1/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.1')
|
||||
updater.run_update()
|
||||
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
pip3 install scrapy
|
||||
pip3 install scrapy-splash
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.1/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.2')
|
||||
updater.run_update()
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo -e $GREEN"Installing YARA ..."$DEFAULT
|
||||
pip3 install yara-python
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.2/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.3')
|
||||
updater.run_update()
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
# echo ""
|
||||
# echo -e $GREEN"installing KVORCKS ..."$DEFAULT
|
||||
# cd ${AIL_HOME}
|
||||
# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git
|
||||
# pushd kvrocks/
|
||||
# make -j4
|
||||
# popd
|
||||
|
||||
echo -e $GREEN"Installing html2text ..."$DEFAULT
|
||||
pip3 install html2text
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.3/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
self.r_serv_onion = self.config.get_redis_conn("ARDB_Onion")
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
Update Domain Languages
|
||||
"""
|
||||
self.r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up')
|
||||
self.r_serv.set('update:nb_elem_to_convert', self.r_serv_onion.scard('domain_update_v3.4'))
|
||||
self.r_serv.set('update:nb_elem_converted', 0)
|
||||
|
||||
# Add background update
|
||||
self.r_serv.sadd('ail:to_update', self.version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.4')
|
||||
updater.run_update()
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
# wait
|
||||
# echo ""
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
# echo ""
|
||||
# echo -e $GREEN"installing KVORCKS ..."$DEFAULT
|
||||
# cd ${AIL_HOME}
|
||||
# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git
|
||||
# pushd kvrocks/
|
||||
# make -j4
|
||||
# popd
|
||||
|
||||
echo -e $GREEN"Installing html2text ..."$DEFAULT
|
||||
pip3 install pycld3
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.4/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
|
||||
def get_domain_type(domain_name):
|
||||
if str(domain_name).endswith('.onion'):
|
||||
return 'onion'
|
||||
else:
|
||||
return 'regular'
|
||||
|
||||
def add_domain_language(domain_name, language):
|
||||
language = language.split('-')[0]
|
||||
domain_type = get_domain_type(domain_name)
|
||||
r_serv_onion.sadd('all_domains_languages', language)
|
||||
r_serv_onion.sadd(f'all_domains_languages:{domain_type}', language)
|
||||
r_serv_onion.sadd(f'language:domains:{domain_type}:{language}', domain_name)
|
||||
r_serv_onion.sadd(f'domain:language:{domain_name}', language)
|
||||
|
||||
def add_domain_languages_by_item_id(domain_name, item_id):
|
||||
item = Item(item_id)
|
||||
for lang in item.get_languages():
|
||||
add_domain_language(domain_name, lang.language)
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
def update_domain_language(domain_obj, item_id):
|
||||
domain_name = domain_obj.get_domain_name()
|
||||
add_domain_languages_by_item_id(domain_name, item_id)
|
||||
|
||||
def get_domain_history(domain_type, domain_name):
|
||||
return r_serv_onion.zrange(f'crawler_history_{domain_type}:{domain_name}:80', 0, -1, withscores=True)
|
||||
|
||||
|
||||
def get_item_children(item_id):
|
||||
return r_serv_metadata.smembers(f'paste_children:{item_id}')
|
||||
|
||||
def get_domain_items(domain_name, root_item_id):
|
||||
dom_item = get_domain_item_children(domain_name, root_item_id)
|
||||
dom_item.append(root_item_id)
|
||||
return dom_item
|
||||
|
||||
def is_item_in_domain(domain_name, item_id):
|
||||
is_in_domain = False
|
||||
domain_length = len(domain_name)
|
||||
if len(item_id) > (domain_length+48):
|
||||
if item_id[-36-domain_length:-36] == domain_name:
|
||||
is_in_domain = True
|
||||
return is_in_domain
|
||||
|
||||
def get_domain_item_children(domain_name, root_item_id):
|
||||
all_items = []
|
||||
for item_id in get_item_children(root_item_id):
|
||||
if is_item_in_domain(domain_name, item_id):
|
||||
all_items.append(item_id)
|
||||
all_items.extend(get_domain_item_children(domain_name, item_id))
|
||||
return all_items
|
||||
|
||||
def get_domain_crawled_item_root(domain_name, domain_type, epoch):
|
||||
res = r_serv_onion.zrevrangebyscore(f'crawler_history_{domain_type}:{domain_name}:80', int(epoch), int(epoch))
|
||||
return {"root_item": res[0], "epoch": int(epoch)}
|
||||
|
||||
def get_domain_items_crawled(domain_name, domain_type, epoch):
|
||||
item_crawled = []
|
||||
item_root = get_domain_crawled_item_root(domain_name, domain_type, epoch)
|
||||
if item_root:
|
||||
if item_root['root_item'] != str(item_root['epoch']) and item_root['root_item']:
|
||||
for item_id in get_domain_items(domain_name, item_root['root_item']):
|
||||
item_crawled.append(item_id)
|
||||
return item_crawled
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
r_serv_db.set('ail:current_background_script', 'domain languages update')
|
||||
|
||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||
if not nb_elem_to_update:
|
||||
nb_elem_to_update = 1
|
||||
else:
|
||||
nb_elem_to_update = int(nb_elem_to_update)
|
||||
|
||||
# _delete_all_domains_languages()
|
||||
|
||||
while True:
|
||||
domain = r_serv_onion.spop('domain_update_v3.4')
|
||||
if domain is not None:
|
||||
print(domain)
|
||||
domain = str(domain)
|
||||
domain_t = get_domain_type(domain)
|
||||
for domain_history in get_domain_history(domain_t, domain):
|
||||
domain_items = get_domain_items_crawled(domain, domain_t, domain_history[1])
|
||||
for id_item in domain_items:
|
||||
update_domain_language(domain, id_item)
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
else:
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
sys.exit(0)
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.5')
|
||||
updater.run_update()
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo -e $GREEN"Installing PyAIL ..."$DEFAULT
|
||||
pip3 install -U pyail
|
||||
|
||||
echo -e $GREEN"Installing D4 CLIENT ..."$DEFAULT
|
||||
pip3 install -U d4-pyclient
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating DomainClassifier ..."$DEFAULT
|
||||
pip3 install -U DomainClassifier
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.6')
|
||||
updater.run_update()
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo -e $GREEN"Updating D4 CLIENT ..."$DEFAULT
|
||||
pip3 install -U d4-pyclient
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Installing nose ..."$DEFAULT
|
||||
pip3 install -U nose
|
||||
|
||||
echo -e $GREEN"Installing coverage ..."$DEFAULT
|
||||
pip3 install -U coverage
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating DomainClassifier ..."$DEFAULT
|
||||
pip3 install -U DomainClassifier
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
from lib import Tracker
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
Update Domain Languages
|
||||
"""
|
||||
print('Fixing Tracker_uuid list ...')
|
||||
Tracker.fix_all_tracker_uuid_list()
|
||||
nb = 0
|
||||
for tracker_uuid in Tracker.get_trackers():
|
||||
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
|
||||
nb += 1
|
||||
|
||||
self.r_serv.set('update:nb_elem_to_convert', nb)
|
||||
self.r_serv.set('update:nb_elem_converted',0)
|
||||
|
||||
# Add background update
|
||||
self.r_serv.sadd('ail:to_update', self.version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v3.7')
|
||||
updater.run_update()
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
|
||||
echo -e $GREEN"Updating thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ut
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.7/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib import Tracker
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tracker = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
Tracker.r_serv_tracker = r_serv_tracker
|
||||
|
||||
r_serv_db.set('ail:current_background_script', 'trackers update')
|
||||
|
||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||
if not nb_elem_to_update:
|
||||
nb_elem_to_update = 1
|
||||
else:
|
||||
nb_elem_to_update = int(nb_elem_to_update)
|
||||
|
||||
while True:
|
||||
tracker_uuid = r_serv_db.spop('trackers_update_v3.7')
|
||||
if tracker_uuid is not None:
|
||||
print(tracker_uuid)
|
||||
# FIX STATS
|
||||
Tracker.fix_tracker_stats_per_day(tracker_uuid)
|
||||
# MAP TRACKER - ITEM_ID
|
||||
Tracker.fix_tracker_item_link(tracker_uuid)
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
else:
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
sys.exit(0)
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.0')
|
||||
updater.run_update()
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Installing nose ..."$DEFAULT
|
||||
pip3 install -U websockets
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
def update(self):
|
||||
r_tracking = redis.StrictRedis(host='localhost',
|
||||
port=6382,
|
||||
db=2,
|
||||
decode_responses=True)
|
||||
# FLUSH OLD DB
|
||||
r_tracking.flushdb()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.1')
|
||||
updater.run_update()
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.2.1')
|
||||
updater.run_update()
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating pusblogger ..."$DEFAULT
|
||||
pip3 install -U pubsublogger
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_HOME'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from update.bin.old_ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
updater = Updater('v4.2')
|
||||
updater.run_update()
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Installing typo-squatting ..."$DEFAULT
|
||||
pip3 install -U ail_typo_squatting
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating d4-client ..."$DEFAULT
|
||||
pip3 install -U d4-pyclient
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in a new issue