AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [Retro Hunt] fix item directory

Browse source
This commit is contained in:
Terrtia 2022-06-16 16:58:07 +02:00
parent 1ddf5a73ce
commit e54cbb3114
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
2 changed files with 28 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
bin/lib/Tracker.py
View file

@ -16,6 +16,7 @@ from flask import escape
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -26,6 +27,8 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
items_dir = config_loader.get_config_str("Directories", "pastes").replace('/', '')
config_loader = None
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
@ -248,7 +251,6 @@ def update_tracker_daterange(tracker_uuid, date, op='add'):
if op == 'del':
pass
def remove_tracked_item(item_id):
item_date = item_basic.get_item_date(item_id)
for tracker_uuid in get_item_all_trackers_uuid(item_id):
@ -267,6 +269,11 @@ def is_obj_tracked(obj_type, subtype, id):
def get_obj_all_trackers(obj_type, subtype, id):
return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{obj_id}')
# # TODO: ADD all Objects + Subtypes
def delete_obj_trackers(obj_type, subtype, id):
if obj_type == 'item':
remove_tracked_item(id)
def get_email_subject(tracker_uuid):
tracker_description = get_tracker_description(tracker_uuid)
if not tracker_description:
@ -1039,7 +1046,7 @@ def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, source
# # TODO: move me
def get_items_to_analyze(dir, last=None):
full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) # # TODO: # FIXME: use item config dir
full_dir = os.path.join(os.environ['AIL_HOME'], items_dir, dir)
if os.path.isdir(full_dir):
all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))])
# remove processed items
@ -1265,7 +1272,21 @@ def api_delete_retro_hunt_task(task_uuid):
else:
return (delete_retro_hunt_task(task_uuid), 200)
# if __name__ == '__main__':
#### DB FIX ####
def get_trackers_custom_tags():
tags = set()
for tracker_uuid in get_all_tracker_uuid():
for tag in get_tracker_tags(tracker_uuid):
tags.add(tag)
for task_uuid in get_all_retro_hunt_tasks():
for tag in get_retro_hunt_task_tags(task_uuid):
tags.add(tag)
return tags
#### -- ####
if __name__ == '__main__':
print(get_trackers_custom_tags())
# fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid()
# print(len(res))

10
bin/trackers/Retro_Hunt.py
View file

@ -65,11 +65,9 @@ class Retro_Hunt(AbstractModule):
# First launch
# restart
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
rule_str = Tracker.get_retro_hunt_task_rule(task_uuid)
self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {rule_str}')
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
@ -90,7 +88,7 @@ class Retro_Hunt(AbstractModule):
# # TODO: Filter previous item
for dir in dirs_date:
print(dir)
self.redis_logger.warning(f'{self.module_name}, Retro Hunt searching in directory {dir}')
self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
l_obj = Tracker.get_items_to_analyze(dir)
for id in l_obj:
#print(f'{dir} / {id}')
@ -98,7 +96,7 @@ class Retro_Hunt(AbstractModule):
# save current item in cache
Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
@ -147,7 +145,7 @@ class Retro_Hunt(AbstractModule):
#print(data)
task_uuid = data['namespace']
self.redis_logger.warning(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}')
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}')
print(f'Retro hunt {task_uuid} match found: {id}')
Tracker.save_retro_hunt_match(task_uuid, id)