From 564280935be8e79c6fdbac6a70c5f1d0363bf3a4 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 27 Aug 2021 18:05:21 +0200 Subject: [PATCH] chg: [tracker + update] add update v3.7 + add map item_id:tracker_uuid (data retention) + fix tracker first_seen/last_seen --- bin/lib/Tracker.py | 123 +++++++++++++++--- bin/packages/Term.py | 18 ++- bin/update-background.py | 1 + update/v3.7/Update.py | 44 +++++++ update/v3.7/Update.sh | 44 +++++++ update/v3.7/Update_trackers.py | 70 ++++++++++ var/www/modules/Flask_config.py | 4 +- .../modules/hunter/templates/showTracker.html | 8 +- 8 files changed, 288 insertions(+), 24 deletions(-) create mode 100755 update/v3.7/Update.py create mode 100755 update/v3.7/Update.sh create mode 100755 update/v3.7/Update_trackers.py diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index aa26abc7..d2350cc9 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -67,6 +67,24 @@ def verify_mail_list(mail_list): ##-- UTILS --## ############### +def get_all_tracker_type(): + return ['word', 'set', 'regex', 'yara'] + +def get_all_tracker_uuid(): + return r_serv_tracker.smembers(f'trackers:all') + +def get_all_tracker_by_type(tracker_type): + r_serv_tracker.smembers(f'trackers:all:{tracker_type}') + +# def get_all_tracker(): +# l_keys_name = [] +# for tracker_type in get_all_tracker_type(): +# l_keys_name.append(f'all:tracker:{tracker_type}') +# return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:]) + +def get_all_tracker_by_type(tracker_type): + return r_serv_tracker.smembers(f'all:tracker:{tracker_type}') + def get_tracker_by_uuid(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked') @@ -94,17 +112,20 @@ def get_tracker_uuid_sources(tracker_uuid): def get_tracker_description(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description') +def get_tracker_date(tracker_uuid): + return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date') + def get_tracker_first_seen(tracker_uuid): - res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0) + res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen') if res: - return res[0] + return res else: return None def get_tracker_last_seen(tracker_uuid): - res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0) + res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen') if res: - return res[0] + return res else: return None @@ -112,7 +133,7 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F dict_uuid = {} dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid) dict_uuid['type'] = get_tracker_type(tracker_uuid) - dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date') + dict_uuid['date'] = get_tracker_date(tracker_uuid) dict_uuid['description'] = get_tracker_description(tracker_uuid) dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid) dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid) @@ -131,15 +152,12 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F dict_uuid['uuid'] = tracker_uuid return dict_uuid -################################################################################ -################################################################################ -################################################################################ -# # TODO: FIXME +# tracker sparkline def get_tracker_sparkline(tracker_uuid, num_day=6): date_range_sparkline = Date.get_date_range(num_day) sparklines_value = [] for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) + nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day)) if nb_seen_this_day is None: nb_seen_this_day = 0 sparklines_value.append(int(nb_seen_this_day)) @@ -159,11 +177,40 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to): def add_tracked_item(tracker_uuid, item_id): item_date = item_basic.get_item_date(item_id) # track item - # r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) + r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id) # track nb item by date if res == 1: - r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1) + nb_items = r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1) + if nb_items == 1: + update_tracker_daterange(tracker_uuid, item_date) + +def set_tracker_first_seen(tracker_uuid, date): + r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date)) + +def set_tracker_last_seen(tracker_uuid, date): + r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date)) + +# # TODO: ADD CACHE ??? +def update_tracker_daterange(tracker_uuid, date, op='add'): + date = int(date) + first_seen = get_tracker_first_seen(tracker_uuid) + + if op == 'add': + if not first_seen: + set_tracker_first_seen(tracker_uuid, date) + set_tracker_last_seen(tracker_uuid, date) + else: + first_seen = int(first_seen) + last_seen = int(get_tracker_last_seen(tracker_uuid)) + if date < first_seen: + set_tracker_first_seen(tracker_uuid, date) + if date > last_seen: + set_tracker_last_seen(tracker_uuid, date) + + if op == 'del': + pass + def remove_tracked_item(item_id): item_date = item_basic.get_item_date(item_id) @@ -237,13 +284,40 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): #### FIX DB #### def fix_tracker_stats_per_day(tracker_uuid): - date_from = get_tracker_first_seen(tracker_uuid) - date_to = get_tracker_last_seen(tracker_uuid) + date_from = get_tracker_date(tracker_uuid) + date_to = Date.get_today_date_str() # delete stats r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}') # create new stats for date_day in Date.substract_date(date_from, date_to): - pass + date_day = int(date_day) + + nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}') + if nb_items: + r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(date_day), nb_items) + + # update first_seen/last_seen + update_tracker_daterange(tracker_uuid, date_day) + +def fix_tracker_item_link(tracker_uuid): + date_from = get_tracker_first_seen(tracker_uuid) + date_to = get_tracker_last_seen(tracker_uuid) + + for date_day in Date.substract_date(date_from, date_to): + l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') + for item_id in l_items: + r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) + +def fix_all_tracker_uuid_list(): + r_serv_tracker.delete(f'trackers:all') + r_serv_tracker.delete(f'trackers:all:{tracker_type}') + for tracker_type in get_all_tracker_type(): + l_tracker = get_all_tracker_by_type(tracker_type) + for tracker in l_tracker: + l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type) + for tracker_uuid in l_tracker_uuid: + r_serv_tracker.sadd(f'trackers:all', tracker_uuid) + r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) ##-- FIX DB --## @@ -359,6 +433,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti # create tracker - uuid map r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid) + r_serv_tracker.sadd(f'trackers:all', tracker_uuid) + r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) + # add display level set if level == 0: # user only r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid) @@ -1121,7 +1198,21 @@ def api_delete_retro_hunt_task(task_uuid): else: return (delete_retro_hunt_task(task_uuid), 200) -#if __name__ == '__main__': +# if __name__ == '__main__': + # fix_all_tracker_uuid_list() + # res = get_all_tracker_uuid() + # print(len(res)) + + # import Term + # Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4') + + #fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4') + + # tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4' + # fix_tracker_item_link(tracker_uuid) + # res = get_item_all_trackers_uuid('archive/') + # print(res) + #res = is_valid_yara_rule('rule dummy { }') # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com']) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 33ad9e76..c8389ab8 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -290,11 +290,16 @@ def parse_tracked_term_to_delete(dict_input, user_id): delete_term(term_uuid) return ({"uuid": term_uuid}, 200) +# # TODO: MOVE IN TRACKER def delete_term(term_uuid): term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) + + r_serv_term.srem(f'trackers:all', term_uuid) + r_serv_term.srem(f'trackers:all:{term_type}', term_uuid) + # Term not tracked by other users if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)): r_serv_term.srem('all:tracker:{}'.format(term_type), term) @@ -323,10 +328,15 @@ def delete_term(term_uuid): r_serv_term.delete('tracker:sources:{}'.format(term_uuid)) # remove item set - all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1) - for date in all_item_date: - r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date)) - r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) + #########################3 + all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True) + if all_item_date: + all_item_date = dict(all_item_date) + for date in all_item_date: + for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'): + r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid) + r_serv_term.delete(f'tracker:item:{term_uuid}:{date}') + r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) if term_type == 'yara': # delete custom rule diff --git a/bin/update-background.py b/bin/update-background.py index d4e7b359..9acc086a 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -65,3 +65,4 @@ if __name__ == "__main__": launch_background_upgrade('v2.6', ['Update_screenshots.py']) launch_background_upgrade('v2.7', ['Update_domain_tags.py']) launch_background_upgrade('v3.4', ['Update_domain.py']) + launch_background_upgrade('v3.7', ['Update_trackers.py']) diff --git a/update/v3.7/Update.py b/update/v3.7/Update.py new file mode 100755 index 00000000..e507b8c2 --- /dev/null +++ b/update/v3.7/Update.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader +import Tracker + +sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) +from ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) + + def update(self): + """ + Update Domain Languages + """ + print('Fixing Tracker_uuid list ...') + Tracker.fix_all_tracker_uuid_list() + nb = 0 + for tracker_uuid in get_all_tracker_uuid: + self.r_serv.sadd('trackers_update_v3.7', tracker_uuid) + nb += 1 + + self.r_serv.set('update:nb_elem_to_convert', nb) + self.r_serv.set('update:nb_elem_converted',0) + + # Add background update + self.r_serv.sadd('ail:to_update', self.version) + +if __name__ == '__main__': + + updater = Updater('v3.7') + updater.run_update() diff --git a/update/v3.7/Update.sh b/update/v3.7/Update.sh new file mode 100755 index 00000000..0196d872 --- /dev/null +++ b/update/v3.7/Update.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +# SUBMODULES # +git submodule update + + +echo -e $GREEN"Updating thirdparty ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ut +wait + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v3.7/Update.py +wait +echo "" +echo "" + + +echo "" +echo -e $GREEN"Shutting down ARDB ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +exit 0 diff --git a/update/v3.7/Update_trackers.py b/update/v3.7/Update_trackers.py new file mode 100755 index 00000000..d67e49d1 --- /dev/null +++ b/update/v3.7/Update_trackers.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader +import Domain + +def update_update_stats(): + nb_updated = int(r_serv_db.get('update:nb_elem_converted')) + progress = int((nb_updated * 100) / nb_elem_to_update) + print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) + r_serv_db.set('ail:current_background_script_stat', progress) + +def update_domain_language(domain_obj, item_id): + domain_name = domain_obj.get_domain_name() + Domain.add_domain_languages_by_item_id(domain_name, item_id) + +if __name__ == '__main__': + + start_deb = time.time() + + config_loader = ConfigLoader.ConfigLoader() + r_serv_db = config_loader.get_redis_conn("ARDB_DB") + r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") + config_loader = None + + r_serv_db.set('ail:current_background_script', 'trackers update') + + nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') + if not nb_elem_to_update: + nb_elem_to_update = 1 + else: + nb_elem_to_update = int(nb_elem_to_update) + + + + while True: + tracker_uuid = r_serv_onion.spop('trackers_update_v3.7') + if tracker_uuid is not None: + date_from = + date_to = + + # FIX STATS + + print(tracker_uuid) + # get all dates + + # get items id + + # convert + domain = Domain.Domain(domain) + for domain_history in domain.get_domain_history(): + domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag + if "items" in domain_item: + for item_dict in domain_item['items']: + update_domain_language(domain, item_dict['id']) + + r_serv_db.incr('update:nb_elem_converted') + update_update_stats() + + else: + r_serv_db.set('ail:current_background_script_stat', 100) + sys.exit(0) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 04336d4e..fd9b0a82 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -109,7 +109,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me 'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be', 'update_warning_message_notice_me': 'missing from the UI.'}, 'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be', - 'update_warning_message_notice_me': 'missing from the UI.'} + 'update_warning_message_notice_me': 'missing from the UI.'}, + 'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be', + 'update_warning_message_notice_me': 'missing from the UI.'} } UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted') diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index bcaeb7dc..0faf20e6 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -209,13 +209,15 @@
- +
- +
@@ -270,7 +272,7 @@