mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [tracker + update] add update v3.7 + add map item_id:tracker_uuid (data retention) + fix tracker first_seen/last_seen
This commit is contained in:
parent
92cfe300f8
commit
564280935b
8 changed files with 288 additions and 24 deletions
|
@ -67,6 +67,24 @@ def verify_mail_list(mail_list):
|
||||||
##-- UTILS --##
|
##-- UTILS --##
|
||||||
###############
|
###############
|
||||||
|
|
||||||
|
def get_all_tracker_type():
|
||||||
|
return ['word', 'set', 'regex', 'yara']
|
||||||
|
|
||||||
|
def get_all_tracker_uuid():
|
||||||
|
return r_serv_tracker.smembers(f'trackers:all')
|
||||||
|
|
||||||
|
def get_all_tracker_by_type(tracker_type):
|
||||||
|
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||||
|
|
||||||
|
# def get_all_tracker():
|
||||||
|
# l_keys_name = []
|
||||||
|
# for tracker_type in get_all_tracker_type():
|
||||||
|
# l_keys_name.append(f'all:tracker:{tracker_type}')
|
||||||
|
# return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:])
|
||||||
|
|
||||||
|
def get_all_tracker_by_type(tracker_type):
|
||||||
|
return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
|
||||||
|
|
||||||
def get_tracker_by_uuid(tracker_uuid):
|
def get_tracker_by_uuid(tracker_uuid):
|
||||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
||||||
|
|
||||||
|
@ -94,17 +112,20 @@ def get_tracker_uuid_sources(tracker_uuid):
|
||||||
def get_tracker_description(tracker_uuid):
|
def get_tracker_description(tracker_uuid):
|
||||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
||||||
|
|
||||||
|
def get_tracker_date(tracker_uuid):
|
||||||
|
return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date')
|
||||||
|
|
||||||
def get_tracker_first_seen(tracker_uuid):
|
def get_tracker_first_seen(tracker_uuid):
|
||||||
res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen')
|
||||||
if res:
|
if res:
|
||||||
return res[0]
|
return res
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_tracker_last_seen(tracker_uuid):
|
def get_tracker_last_seen(tracker_uuid):
|
||||||
res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen')
|
||||||
if res:
|
if res:
|
||||||
return res[0]
|
return res
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -112,7 +133,7 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
|
||||||
dict_uuid = {}
|
dict_uuid = {}
|
||||||
dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
|
dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
|
||||||
dict_uuid['type'] = get_tracker_type(tracker_uuid)
|
dict_uuid['type'] = get_tracker_type(tracker_uuid)
|
||||||
dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date')
|
dict_uuid['date'] = get_tracker_date(tracker_uuid)
|
||||||
dict_uuid['description'] = get_tracker_description(tracker_uuid)
|
dict_uuid['description'] = get_tracker_description(tracker_uuid)
|
||||||
dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
|
dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
|
||||||
dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
|
dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
|
||||||
|
@ -131,15 +152,12 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
|
||||||
dict_uuid['uuid'] = tracker_uuid
|
dict_uuid['uuid'] = tracker_uuid
|
||||||
return dict_uuid
|
return dict_uuid
|
||||||
|
|
||||||
################################################################################
|
# tracker sparkline
|
||||||
################################################################################
|
|
||||||
################################################################################
|
|
||||||
# # TODO: FIXME
|
|
||||||
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
||||||
date_range_sparkline = Date.get_date_range(num_day)
|
date_range_sparkline = Date.get_date_range(num_day)
|
||||||
sparklines_value = []
|
sparklines_value = []
|
||||||
for date_day in date_range_sparkline:
|
for date_day in date_range_sparkline:
|
||||||
nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
|
nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day))
|
||||||
if nb_seen_this_day is None:
|
if nb_seen_this_day is None:
|
||||||
nb_seen_this_day = 0
|
nb_seen_this_day = 0
|
||||||
sparklines_value.append(int(nb_seen_this_day))
|
sparklines_value.append(int(nb_seen_this_day))
|
||||||
|
@ -159,11 +177,40 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
||||||
def add_tracked_item(tracker_uuid, item_id):
|
def add_tracked_item(tracker_uuid, item_id):
|
||||||
item_date = item_basic.get_item_date(item_id)
|
item_date = item_basic.get_item_date(item_id)
|
||||||
# track item
|
# track item
|
||||||
# r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
||||||
res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
|
res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
|
||||||
# track nb item by date
|
# track nb item by date
|
||||||
if res == 1:
|
if res == 1:
|
||||||
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
|
nb_items = r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
|
||||||
|
if nb_items == 1:
|
||||||
|
update_tracker_daterange(tracker_uuid, item_date)
|
||||||
|
|
||||||
|
def set_tracker_first_seen(tracker_uuid, date):
|
||||||
|
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date))
|
||||||
|
|
||||||
|
def set_tracker_last_seen(tracker_uuid, date):
|
||||||
|
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date))
|
||||||
|
|
||||||
|
# # TODO: ADD CACHE ???
|
||||||
|
def update_tracker_daterange(tracker_uuid, date, op='add'):
|
||||||
|
date = int(date)
|
||||||
|
first_seen = get_tracker_first_seen(tracker_uuid)
|
||||||
|
|
||||||
|
if op == 'add':
|
||||||
|
if not first_seen:
|
||||||
|
set_tracker_first_seen(tracker_uuid, date)
|
||||||
|
set_tracker_last_seen(tracker_uuid, date)
|
||||||
|
else:
|
||||||
|
first_seen = int(first_seen)
|
||||||
|
last_seen = int(get_tracker_last_seen(tracker_uuid))
|
||||||
|
if date < first_seen:
|
||||||
|
set_tracker_first_seen(tracker_uuid, date)
|
||||||
|
if date > last_seen:
|
||||||
|
set_tracker_last_seen(tracker_uuid, date)
|
||||||
|
|
||||||
|
if op == 'del':
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def remove_tracked_item(item_id):
|
def remove_tracked_item(item_id):
|
||||||
item_date = item_basic.get_item_date(item_id)
|
item_date = item_basic.get_item_date(item_id)
|
||||||
|
@ -237,13 +284,40 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
||||||
|
|
||||||
#### FIX DB ####
|
#### FIX DB ####
|
||||||
def fix_tracker_stats_per_day(tracker_uuid):
|
def fix_tracker_stats_per_day(tracker_uuid):
|
||||||
date_from = get_tracker_first_seen(tracker_uuid)
|
date_from = get_tracker_date(tracker_uuid)
|
||||||
date_to = get_tracker_last_seen(tracker_uuid)
|
date_to = Date.get_today_date_str()
|
||||||
# delete stats
|
# delete stats
|
||||||
r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}')
|
r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}')
|
||||||
# create new stats
|
# create new stats
|
||||||
for date_day in Date.substract_date(date_from, date_to):
|
for date_day in Date.substract_date(date_from, date_to):
|
||||||
pass
|
date_day = int(date_day)
|
||||||
|
|
||||||
|
nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||||
|
if nb_items:
|
||||||
|
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(date_day), nb_items)
|
||||||
|
|
||||||
|
# update first_seen/last_seen
|
||||||
|
update_tracker_daterange(tracker_uuid, date_day)
|
||||||
|
|
||||||
|
def fix_tracker_item_link(tracker_uuid):
|
||||||
|
date_from = get_tracker_first_seen(tracker_uuid)
|
||||||
|
date_to = get_tracker_last_seen(tracker_uuid)
|
||||||
|
|
||||||
|
for date_day in Date.substract_date(date_from, date_to):
|
||||||
|
l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||||
|
for item_id in l_items:
|
||||||
|
r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
||||||
|
|
||||||
|
def fix_all_tracker_uuid_list():
|
||||||
|
r_serv_tracker.delete(f'trackers:all')
|
||||||
|
r_serv_tracker.delete(f'trackers:all:{tracker_type}')
|
||||||
|
for tracker_type in get_all_tracker_type():
|
||||||
|
l_tracker = get_all_tracker_by_type(tracker_type)
|
||||||
|
for tracker in l_tracker:
|
||||||
|
l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type)
|
||||||
|
for tracker_uuid in l_tracker_uuid:
|
||||||
|
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
|
||||||
|
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
|
||||||
|
|
||||||
##-- FIX DB --##
|
##-- FIX DB --##
|
||||||
|
|
||||||
|
@ -359,6 +433,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
||||||
# create tracker - uuid map
|
# create tracker - uuid map
|
||||||
r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
|
r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
|
||||||
|
|
||||||
|
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
|
||||||
|
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
|
||||||
|
|
||||||
# add display level set
|
# add display level set
|
||||||
if level == 0: # user only
|
if level == 0: # user only
|
||||||
r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
|
r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
|
||||||
|
@ -1121,7 +1198,21 @@ def api_delete_retro_hunt_task(task_uuid):
|
||||||
else:
|
else:
|
||||||
return (delete_retro_hunt_task(task_uuid), 200)
|
return (delete_retro_hunt_task(task_uuid), 200)
|
||||||
|
|
||||||
#if __name__ == '__main__':
|
# if __name__ == '__main__':
|
||||||
|
# fix_all_tracker_uuid_list()
|
||||||
|
# res = get_all_tracker_uuid()
|
||||||
|
# print(len(res))
|
||||||
|
|
||||||
|
# import Term
|
||||||
|
# Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
||||||
|
|
||||||
|
#fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
||||||
|
|
||||||
|
# tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
|
||||||
|
# fix_tracker_item_link(tracker_uuid)
|
||||||
|
# res = get_item_all_trackers_uuid('archive/')
|
||||||
|
# print(res)
|
||||||
|
|
||||||
#res = is_valid_yara_rule('rule dummy { }')
|
#res = is_valid_yara_rule('rule dummy { }')
|
||||||
|
|
||||||
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
||||||
|
|
|
@ -290,11 +290,16 @@ def parse_tracked_term_to_delete(dict_input, user_id):
|
||||||
delete_term(term_uuid)
|
delete_term(term_uuid)
|
||||||
return ({"uuid": term_uuid}, 200)
|
return ({"uuid": term_uuid}, 200)
|
||||||
|
|
||||||
|
# # TODO: MOVE IN TRACKER
|
||||||
def delete_term(term_uuid):
|
def delete_term(term_uuid):
|
||||||
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
||||||
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
||||||
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||||
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
|
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
|
||||||
|
|
||||||
|
r_serv_term.srem(f'trackers:all', term_uuid)
|
||||||
|
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
|
||||||
|
|
||||||
# Term not tracked by other users
|
# Term not tracked by other users
|
||||||
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
|
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
|
||||||
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
|
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
|
||||||
|
@ -323,10 +328,15 @@ def delete_term(term_uuid):
|
||||||
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
||||||
|
|
||||||
# remove item set
|
# remove item set
|
||||||
all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1)
|
#########################3
|
||||||
for date in all_item_date:
|
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
|
||||||
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
|
if all_item_date:
|
||||||
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
all_item_date = dict(all_item_date)
|
||||||
|
for date in all_item_date:
|
||||||
|
for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
|
||||||
|
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
|
||||||
|
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
|
||||||
|
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
||||||
|
|
||||||
if term_type == 'yara':
|
if term_type == 'yara':
|
||||||
# delete custom rule
|
# delete custom rule
|
||||||
|
|
|
@ -65,3 +65,4 @@ if __name__ == "__main__":
|
||||||
launch_background_upgrade('v2.6', ['Update_screenshots.py'])
|
launch_background_upgrade('v2.6', ['Update_screenshots.py'])
|
||||||
launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
|
launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
|
||||||
launch_background_upgrade('v3.4', ['Update_domain.py'])
|
launch_background_upgrade('v3.4', ['Update_domain.py'])
|
||||||
|
launch_background_upgrade('v3.7', ['Update_trackers.py'])
|
||||||
|
|
44
update/v3.7/Update.py
Executable file
44
update/v3.7/Update.py
Executable file
|
@ -0,0 +1,44 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
import Tracker
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||||
|
from ail_updater import AIL_Updater
|
||||||
|
|
||||||
|
class Updater(AIL_Updater):
|
||||||
|
"""default Updater."""
|
||||||
|
|
||||||
|
def __init__(self, version):
|
||||||
|
super(Updater, self).__init__(version)
|
||||||
|
|
||||||
|
def update(self):
|
||||||
|
"""
|
||||||
|
Update Domain Languages
|
||||||
|
"""
|
||||||
|
print('Fixing Tracker_uuid list ...')
|
||||||
|
Tracker.fix_all_tracker_uuid_list()
|
||||||
|
nb = 0
|
||||||
|
for tracker_uuid in get_all_tracker_uuid:
|
||||||
|
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
|
||||||
|
nb += 1
|
||||||
|
|
||||||
|
self.r_serv.set('update:nb_elem_to_convert', nb)
|
||||||
|
self.r_serv.set('update:nb_elem_converted',0)
|
||||||
|
|
||||||
|
# Add background update
|
||||||
|
self.r_serv.sadd('ail:to_update', self.version)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
updater = Updater('v3.7')
|
||||||
|
updater.run_update()
|
44
update/v3.7/Update.sh
Executable file
44
update/v3.7/Update.sh
Executable file
|
@ -0,0 +1,44 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||||
|
|
||||||
|
export PATH=$AIL_HOME:$PATH
|
||||||
|
export PATH=$AIL_REDIS:$PATH
|
||||||
|
export PATH=$AIL_ARDB:$PATH
|
||||||
|
export PATH=$AIL_BIN:$PATH
|
||||||
|
export PATH=$AIL_FLASK:$PATH
|
||||||
|
|
||||||
|
GREEN="\\033[1;32m"
|
||||||
|
DEFAULT="\\033[0;39m"
|
||||||
|
|
||||||
|
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
# SUBMODULES #
|
||||||
|
git submodule update
|
||||||
|
|
||||||
|
|
||||||
|
echo -e $GREEN"Updating thirdparty ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ut
|
||||||
|
wait
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||||
|
echo ""
|
||||||
|
python ${AIL_HOME}/update/v3.7/Update.py
|
||||||
|
wait
|
||||||
|
echo ""
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
exit 0
|
70
update/v3.7/Update_trackers.py
Executable file
70
update/v3.7/Update_trackers.py
Executable file
|
@ -0,0 +1,70 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
import Domain
|
||||||
|
|
||||||
|
def update_update_stats():
|
||||||
|
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||||
|
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||||
|
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||||
|
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||||
|
|
||||||
|
def update_domain_language(domain_obj, item_id):
|
||||||
|
domain_name = domain_obj.get_domain_name()
|
||||||
|
Domain.add_domain_languages_by_item_id(domain_name, item_id)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
start_deb = time.time()
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
r_serv_db.set('ail:current_background_script', 'trackers update')
|
||||||
|
|
||||||
|
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||||
|
if not nb_elem_to_update:
|
||||||
|
nb_elem_to_update = 1
|
||||||
|
else:
|
||||||
|
nb_elem_to_update = int(nb_elem_to_update)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
while True:
|
||||||
|
tracker_uuid = r_serv_onion.spop('trackers_update_v3.7')
|
||||||
|
if tracker_uuid is not None:
|
||||||
|
date_from =
|
||||||
|
date_to =
|
||||||
|
|
||||||
|
# FIX STATS
|
||||||
|
|
||||||
|
print(tracker_uuid)
|
||||||
|
# get all dates
|
||||||
|
|
||||||
|
# get items id
|
||||||
|
|
||||||
|
# convert
|
||||||
|
domain = Domain.Domain(domain)
|
||||||
|
for domain_history in domain.get_domain_history():
|
||||||
|
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||||
|
if "items" in domain_item:
|
||||||
|
for item_dict in domain_item['items']:
|
||||||
|
update_domain_language(domain, item_dict['id'])
|
||||||
|
|
||||||
|
r_serv_db.incr('update:nb_elem_converted')
|
||||||
|
update_update_stats()
|
||||||
|
|
||||||
|
else:
|
||||||
|
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||||
|
sys.exit(0)
|
|
@ -109,7 +109,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
|
||||||
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
||||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||||
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
|
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
|
||||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||||
|
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
|
||||||
|
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||||
}
|
}
|
||||||
|
|
||||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||||
|
|
|
@ -209,13 +209,15 @@
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group" id="date-range-from">
|
<div class="input-group" id="date-range-from">
|
||||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_from'] }}" name="date_from" autocomplete="off">
|
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
|
||||||
|
{%if tracker_metadata['date_from']%}value="{{ tracker_metadata['date_from'] }}"{%else%}value="{{tracker_metadata['first_seen']}}"{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group" id="date-range-to">
|
<div class="input-group" id="date-range-to">
|
||||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_to'] }}" name="date_to" autocomplete="off">
|
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
|
||||||
|
{%if tracker_metadata['date_to']%}value="{{ tracker_metadata['date_to'] }}"{%else%}value="{{tracker_metadata['last_seen']}}"{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -270,7 +272,7 @@
|
||||||
<script>
|
<script>
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
$('#div_edit_mails').hide();
|
$('#div_edit_mails').hide();
|
||||||
$('#div_edit_tags').hide();edit_description
|
$('#div_edit_tags').hide();
|
||||||
$('#div_edit_description').hide();
|
$('#div_edit_description').hide();
|
||||||
$("#page-Decoded").addClass("active");
|
$("#page-Decoded").addClass("active");
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue