From e54cbb3114c3c0467c500b175c2a4f9eb62758e9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 16 Jun 2022 16:58:07 +0200 Subject: [PATCH] fix: [Retro Hunt] fix item directory --- bin/lib/Tracker.py | 27 ++++++++++++++++++++++++--- bin/trackers/Retro_Hunt.py | 10 ++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 2f8e8d42..4bbf3f5b 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -16,6 +16,7 @@ from flask import escape sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date +import Tag sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -26,6 +27,8 @@ r_cache = config_loader.get_redis_conn("Redis_Cache") r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") + +items_dir = config_loader.get_config_str("Directories", "pastes").replace('/', '') config_loader = None email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' @@ -248,7 +251,6 @@ def update_tracker_daterange(tracker_uuid, date, op='add'): if op == 'del': pass - def remove_tracked_item(item_id): item_date = item_basic.get_item_date(item_id) for tracker_uuid in get_item_all_trackers_uuid(item_id): @@ -267,6 +269,11 @@ def is_obj_tracked(obj_type, subtype, id): def get_obj_all_trackers(obj_type, subtype, id): return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{obj_id}') +# # TODO: ADD all Objects + Subtypes +def delete_obj_trackers(obj_type, subtype, id): + if obj_type == 'item': + remove_tracked_item(id) + def get_email_subject(tracker_uuid): tracker_description = get_tracker_description(tracker_uuid) if not tracker_description: @@ -1039,7 +1046,7 @@ def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, source # # TODO: move me def get_items_to_analyze(dir, last=None): - full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) # # TODO: # FIXME: use item config dir + full_dir = os.path.join(os.environ['AIL_HOME'], items_dir, dir) if os.path.isdir(full_dir): all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))]) # remove processed items @@ -1265,7 +1272,21 @@ def api_delete_retro_hunt_task(task_uuid): else: return (delete_retro_hunt_task(task_uuid), 200) -# if __name__ == '__main__': +#### DB FIX #### +def get_trackers_custom_tags(): + tags = set() + for tracker_uuid in get_all_tracker_uuid(): + for tag in get_tracker_tags(tracker_uuid): + tags.add(tag) + for task_uuid in get_all_retro_hunt_tasks(): + for tag in get_retro_hunt_task_tags(task_uuid): + tags.add(tag) + return tags + +#### -- #### + +if __name__ == '__main__': + print(get_trackers_custom_tags()) # fix_all_tracker_uuid_list() # res = get_all_tracker_uuid() # print(len(res)) diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index c07f5eea..037520fc 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -65,11 +65,9 @@ class Retro_Hunt(AbstractModule): # First launch # restart rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True) - rule_str = Tracker.get_retro_hunt_task_rule(task_uuid) - self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {rule_str}') timeout = Tracker.get_retro_hunt_task_timeout(task_uuid) - self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') + self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True) self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid) @@ -90,7 +88,7 @@ class Retro_Hunt(AbstractModule): # # TODO: Filter previous item for dir in dirs_date: print(dir) - self.redis_logger.warning(f'{self.module_name}, Retro Hunt searching in directory {dir}') + self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}') l_obj = Tracker.get_items_to_analyze(dir) for id in l_obj: #print(f'{dir} / {id}') @@ -98,7 +96,7 @@ class Retro_Hunt(AbstractModule): # save current item in cache Tracker.set_cache_retro_hunt_task_id(task_uuid, id) - self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}') + self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}') yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) @@ -147,7 +145,7 @@ class Retro_Hunt(AbstractModule): #print(data) task_uuid = data['namespace'] - self.redis_logger.warning(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}') + self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}') print(f'Retro hunt {task_uuid} match found: {id}') Tracker.save_retro_hunt_match(task_uuid, id)