chg: [tracker + update] add update v3.7 + add map item_id:tracker_uuid (data retention) + fix tracker first_seen/last_seen

This commit is contained in:
Terrtia 2021-08-27 18:05:21 +02:00
parent 92cfe300f8
commit 564280935b
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
8 changed files with 288 additions and 24 deletions

View file

@ -67,6 +67,24 @@ def verify_mail_list(mail_list):
##-- UTILS --## ##-- UTILS --##
############### ###############
def get_all_tracker_type():
return ['word', 'set', 'regex', 'yara']
def get_all_tracker_uuid():
return r_serv_tracker.smembers(f'trackers:all')
def get_all_tracker_by_type(tracker_type):
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
# def get_all_tracker():
# l_keys_name = []
# for tracker_type in get_all_tracker_type():
# l_keys_name.append(f'all:tracker:{tracker_type}')
# return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:])
def get_all_tracker_by_type(tracker_type):
return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
def get_tracker_by_uuid(tracker_uuid): def get_tracker_by_uuid(tracker_uuid):
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked') return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
@ -94,17 +112,20 @@ def get_tracker_uuid_sources(tracker_uuid):
def get_tracker_description(tracker_uuid): def get_tracker_description(tracker_uuid):
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description') return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
def get_tracker_date(tracker_uuid):
return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date')
def get_tracker_first_seen(tracker_uuid): def get_tracker_first_seen(tracker_uuid):
res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0) res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen')
if res: if res:
return res[0] return res
else: else:
return None return None
def get_tracker_last_seen(tracker_uuid): def get_tracker_last_seen(tracker_uuid):
res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0) res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen')
if res: if res:
return res[0] return res
else: else:
return None return None
@ -112,7 +133,7 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
dict_uuid = {} dict_uuid = {}
dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid) dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
dict_uuid['type'] = get_tracker_type(tracker_uuid) dict_uuid['type'] = get_tracker_type(tracker_uuid)
dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date') dict_uuid['date'] = get_tracker_date(tracker_uuid)
dict_uuid['description'] = get_tracker_description(tracker_uuid) dict_uuid['description'] = get_tracker_description(tracker_uuid)
dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid) dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid) dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
@ -131,15 +152,12 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
dict_uuid['uuid'] = tracker_uuid dict_uuid['uuid'] = tracker_uuid
return dict_uuid return dict_uuid
################################################################################ # tracker sparkline
################################################################################
################################################################################
# # TODO: FIXME
def get_tracker_sparkline(tracker_uuid, num_day=6): def get_tracker_sparkline(tracker_uuid, num_day=6):
date_range_sparkline = Date.get_date_range(num_day) date_range_sparkline = Date.get_date_range(num_day)
sparklines_value = [] sparklines_value = []
for date_day in date_range_sparkline: for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day))
if nb_seen_this_day is None: if nb_seen_this_day is None:
nb_seen_this_day = 0 nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day)) sparklines_value.append(int(nb_seen_this_day))
@ -159,11 +177,40 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
def add_tracked_item(tracker_uuid, item_id): def add_tracked_item(tracker_uuid, item_id):
item_date = item_basic.get_item_date(item_id) item_date = item_basic.get_item_date(item_id)
# track item # track item
# r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id) res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
# track nb item by date # track nb item by date
if res == 1: if res == 1:
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1) nb_items = r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
if nb_items == 1:
update_tracker_daterange(tracker_uuid, item_date)
def set_tracker_first_seen(tracker_uuid, date):
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date))
def set_tracker_last_seen(tracker_uuid, date):
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date))
# # TODO: ADD CACHE ???
def update_tracker_daterange(tracker_uuid, date, op='add'):
date = int(date)
first_seen = get_tracker_first_seen(tracker_uuid)
if op == 'add':
if not first_seen:
set_tracker_first_seen(tracker_uuid, date)
set_tracker_last_seen(tracker_uuid, date)
else:
first_seen = int(first_seen)
last_seen = int(get_tracker_last_seen(tracker_uuid))
if date < first_seen:
set_tracker_first_seen(tracker_uuid, date)
if date > last_seen:
set_tracker_last_seen(tracker_uuid, date)
if op == 'del':
pass
def remove_tracked_item(item_id): def remove_tracked_item(item_id):
item_date = item_basic.get_item_date(item_id) item_date = item_basic.get_item_date(item_id)
@ -237,13 +284,40 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
#### FIX DB #### #### FIX DB ####
def fix_tracker_stats_per_day(tracker_uuid): def fix_tracker_stats_per_day(tracker_uuid):
date_from = get_tracker_first_seen(tracker_uuid) date_from = get_tracker_date(tracker_uuid)
date_to = get_tracker_last_seen(tracker_uuid) date_to = Date.get_today_date_str()
# delete stats # delete stats
r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}') r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}')
# create new stats # create new stats
for date_day in Date.substract_date(date_from, date_to): for date_day in Date.substract_date(date_from, date_to):
pass date_day = int(date_day)
nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
if nb_items:
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(date_day), nb_items)
# update first_seen/last_seen
update_tracker_daterange(tracker_uuid, date_day)
def fix_tracker_item_link(tracker_uuid):
date_from = get_tracker_first_seen(tracker_uuid)
date_to = get_tracker_last_seen(tracker_uuid)
for date_day in Date.substract_date(date_from, date_to):
l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
for item_id in l_items:
r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
def fix_all_tracker_uuid_list():
r_serv_tracker.delete(f'trackers:all')
r_serv_tracker.delete(f'trackers:all:{tracker_type}')
for tracker_type in get_all_tracker_type():
l_tracker = get_all_tracker_by_type(tracker_type)
for tracker in l_tracker:
l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type)
for tracker_uuid in l_tracker_uuid:
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
##-- FIX DB --## ##-- FIX DB --##
@ -359,6 +433,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
# create tracker - uuid map # create tracker - uuid map
r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid) r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
# add display level set # add display level set
if level == 0: # user only if level == 0: # user only
r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid) r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
@ -1121,7 +1198,21 @@ def api_delete_retro_hunt_task(task_uuid):
else: else:
return (delete_retro_hunt_task(task_uuid), 200) return (delete_retro_hunt_task(task_uuid), 200)
#if __name__ == '__main__': # if __name__ == '__main__':
# fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid()
# print(len(res))
# import Term
# Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
#fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
# tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
# fix_tracker_item_link(tracker_uuid)
# res = get_item_all_trackers_uuid('archive/')
# print(res)
#res = is_valid_yara_rule('rule dummy { }') #res = is_valid_yara_rule('rule dummy { }')
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com']) # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])

View file

@ -290,11 +290,16 @@ def parse_tracked_term_to_delete(dict_input, user_id):
delete_term(term_uuid) delete_term(term_uuid)
return ({"uuid": term_uuid}, 200) return ({"uuid": term_uuid}, 200)
# # TODO: MOVE IN TRACKER
def delete_term(term_uuid): def delete_term(term_uuid):
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
r_serv_term.srem(f'trackers:all', term_uuid)
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
# Term not tracked by other users # Term not tracked by other users
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)): if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
r_serv_term.srem('all:tracker:{}'.format(term_type), term) r_serv_term.srem('all:tracker:{}'.format(term_type), term)
@ -323,9 +328,14 @@ def delete_term(term_uuid):
r_serv_term.delete('tracker:sources:{}'.format(term_uuid)) r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
# remove item set # remove item set
all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1) #########################3
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
if all_item_date:
all_item_date = dict(all_item_date)
for date in all_item_date: for date in all_item_date:
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date)) for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
if term_type == 'yara': if term_type == 'yara':

View file

@ -65,3 +65,4 @@ if __name__ == "__main__":
launch_background_upgrade('v2.6', ['Update_screenshots.py']) launch_background_upgrade('v2.6', ['Update_screenshots.py'])
launch_background_upgrade('v2.7', ['Update_domain_tags.py']) launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
launch_background_upgrade('v3.4', ['Update_domain.py']) launch_background_upgrade('v3.4', ['Update_domain.py'])
launch_background_upgrade('v3.7', ['Update_trackers.py'])

44
update/v3.7/Update.py Executable file
View file

@ -0,0 +1,44 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Tracker
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
def update(self):
"""
Update Domain Languages
"""
print('Fixing Tracker_uuid list ...')
Tracker.fix_all_tracker_uuid_list()
nb = 0
for tracker_uuid in get_all_tracker_uuid:
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
nb += 1
self.r_serv.set('update:nb_elem_to_convert', nb)
self.r_serv.set('update:nb_elem_converted',0)
# Add background update
self.r_serv.sadd('ail:to_update', self.version)
if __name__ == '__main__':
updater = Updater('v3.7')
updater.run_update()

44
update/v3.7/Update.sh Executable file
View file

@ -0,0 +1,44 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
# SUBMODULES #
git submodule update
echo -e $GREEN"Updating thirdparty ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ut
wait
echo ""
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
echo ""
python ${AIL_HOME}/update/v3.7/Update.py
wait
echo ""
echo ""
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
exit 0

70
update/v3.7/Update_trackers.py Executable file
View file

@ -0,0 +1,70 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Domain
def update_update_stats():
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
progress = int((nb_updated * 100) / nb_elem_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
r_serv_db.set('ail:current_background_script_stat', progress)
def update_domain_language(domain_obj, item_id):
domain_name = domain_obj.get_domain_name()
Domain.add_domain_languages_by_item_id(domain_name, item_id)
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None
r_serv_db.set('ail:current_background_script', 'trackers update')
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
if not nb_elem_to_update:
nb_elem_to_update = 1
else:
nb_elem_to_update = int(nb_elem_to_update)
while True:
tracker_uuid = r_serv_onion.spop('trackers_update_v3.7')
if tracker_uuid is not None:
date_from =
date_to =
# FIX STATS
print(tracker_uuid)
# get all dates
# get items id
# convert
domain = Domain.Domain(domain)
for domain_history in domain.get_domain_history():
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
if "items" in domain_item:
for item_dict in domain_item['items']:
update_domain_language(domain, item_dict['id'])
r_serv_db.incr('update:nb_elem_converted')
update_update_stats()
else:
r_serv_db.set('ail:current_background_script_stat', 100)
sys.exit(0)

View file

@ -109,6 +109,8 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be', 'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be', 'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
'update_warning_message_notice_me': 'missing from the UI.'} 'update_warning_message_notice_me': 'missing from the UI.'}
} }

View file

@ -209,13 +209,15 @@
<div class="col-md-6"> <div class="col-md-6">
<div class="input-group" id="date-range-from"> <div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_from'] }}" name="date_from" autocomplete="off"> <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
{%if tracker_metadata['date_from']%}value="{{ tracker_metadata['date_from'] }}"{%else%}value="{{tracker_metadata['first_seen']}}"{%endif%}>
</div> </div>
</div> </div>
<div class="col-md-6"> <div class="col-md-6">
<div class="input-group" id="date-range-to"> <div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_to'] }}" name="date_to" autocomplete="off"> <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
{%if tracker_metadata['date_to']%}value="{{ tracker_metadata['date_to'] }}"{%else%}value="{{tracker_metadata['last_seen']}}"{%endif%}>
</div> </div>
</div> </div>
</div> </div>
@ -270,7 +272,7 @@
<script> <script>
$(document).ready(function(){ $(document).ready(function(){
$('#div_edit_mails').hide(); $('#div_edit_mails').hide();
$('#div_edit_tags').hide();edit_description $('#div_edit_tags').hide();
$('#div_edit_description').hide(); $('#div_edit_description').hide();
$("#page-Decoded").addClass("active"); $("#page-Decoded").addClass("active");