From 37c71b843820b694a544ac6b231306e1af035693 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 10 May 2023 16:26:46 +0200 Subject: [PATCH] chg: [objects + retro hunt] refactor retro hunt + objects retro hunts + get objects generator by filters (date_from, sources, mimetypes, ...) --- bin/lib/Tracker.py | 824 +++++++----------- bin/lib/ail_core.py | 18 + bin/lib/objects/Decodeds.py | 71 +- bin/lib/objects/Items.py | 118 ++- bin/lib/objects/Pgps.py | 18 +- bin/lib/objects/abstract_subtype_object.py | 5 +- bin/lib/objects/ail_objects.py | 34 +- bin/packages/Date.py | 7 +- bin/trackers/Retro_Hunt.py | 160 ++-- configs/modules.cfg | 2 +- var/www/blueprints/hunters.py | 142 +-- var/www/modules/hunter/Flask_hunter.py | 1 + .../templates/hunter/add_retro_hunt_task.html | 268 ++++-- var/www/templates/hunter/show_retro_hunt.html | 240 ++--- 14 files changed, 1026 insertions(+), 882 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index b474116b..fcb222f8 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -23,7 +23,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from packages import Date -from lib.ail_core import get_objects_tracked, get_object_all_subtypes +from lib.ail_core import get_objects_tracked, get_object_all_subtypes, get_objects_retro_hunted from lib import ConfigLoader from lib import item_basic from lib import Tag @@ -34,8 +34,6 @@ r_cache = config_loader.get_redis_conn("Redis_Cache") r_tracker = config_loader.get_db_conn("Kvrocks_Trackers") -r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") # TODO REMOVE ME - items_dir = config_loader.get_config_str("Directories", "pastes") if items_dir[-1] == '/': items_dir = items_dir[:-1] @@ -49,17 +47,17 @@ special_characters.add('\\s') # NLTK tokenizer tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+', - gaps=True, discard_empty=True) + gaps=True, discard_empty=True) ############### #### UTILS #### -def is_valid_uuid_v4(UUID): - if not UUID: +def is_valid_uuid_v4(curr_uuid): + if not curr_uuid: return False - UUID = UUID.replace('-', '') + curr_uuid = curr_uuid.replace('-', '') try: - uuid_test = uuid.UUID(hex=UUID, version=4) - return uuid_test.hex == UUID + uuid_test = uuid.UUID(hex=curr_uuid, version=4) + return uuid_test.hex == curr_uuid except: return False @@ -83,12 +81,11 @@ def verify_mail_list(mail_list): return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400 return None -##-- UTILS --## -############### +## -- UTILS -- ## +################# -################################################################################################ -################################################################################################ -################################################################################################ +################### +#### TRACKER #### class Tracker: def __init__(self, tracker_uuid): @@ -125,7 +122,7 @@ class Tracker: self._set_field('last_seen', date) def _exist_date(self, date): - return r_serv_tracker.exists(f'tracker:objs:{self.uuid}:{date}') + return r_tracker.exists(f'tracker:objs:{self.uuid}:{date}') # TODO: ADD CACHE ??? def update_daterange(self, date=None): @@ -189,11 +186,11 @@ class Tracker: if level == 0: # user only if not user: user = self.get_user() - r_serv_tracker.sadd(f'user:tracker:{user}', self.uuid) - r_serv_tracker.sadd(f'user:tracker:{user}:{tracker_type}', self.uuid) + r_tracker.sadd(f'user:tracker:{user}', self.uuid) + r_tracker.sadd(f'user:tracker:{user}:{tracker_type}', self.uuid) elif level == 1: # global - r_serv_tracker.sadd('global:tracker', self.uuid) - r_serv_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid) + r_tracker.sadd('global:tracker', self.uuid) + r_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid) self._set_field('level', level) def get_filters(self): @@ -219,8 +216,8 @@ class Tracker: def _set_tags(self, tags): for tag in tags: tag = escape(tag) - r_serv_tracker.sadd(f'tracker:tags:{self.uuid}', tag) - Tag.create_custom_tag(tag) # TODO CUSTOM TAGS + r_tracker.sadd(f'tracker:tags:{self.uuid}', tag) + Tag.create_custom_tag(tag) # TODO CUSTOM TAGS def mail_export(self): return r_tracker.exists(f'tracker:mail:{self.uuid}') @@ -230,7 +227,7 @@ class Tracker: def _set_mails(self, mails): for mail in mails: - r_serv_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail)) + r_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail)) def get_user(self): return r_tracker.hget(f'tracker:{self.uuid}', 'user_id') @@ -255,10 +252,6 @@ class Tracker: yar_path = self.get_tracked() return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path)) - # TODO get objects/ tracked items - - - # TODO sparkline def get_meta(self, options): if not options: options = set() @@ -290,11 +283,11 @@ class Tracker: mess = f'{self.uuid}:{int(time.time())}:{obj_type}:{subtype}:{obj_id}' if self.is_level_user(): user = self.get_user() - r_serv_tracker.lpush(f'trackers:user:{user}', mess) - r_serv_tracker.ltrim(f'trackers:user:{user}', 0, 9) + r_tracker.lpush(f'trackers:user:{user}', mess) + r_tracker.ltrim(f'trackers:user:{user}', 0, 9) else: - r_serv_tracker.lpush('trackers:dashboard', mess) - r_serv_tracker.ltrim(f'trackers:dashboard', 0, 9) + r_tracker.lpush('trackers:dashboard', mess) + r_tracker.ltrim(f'trackers:dashboard', 0, 9) # - TODO Data Retention TO Implement - # # Or Daily/Monthly Global DB Cleanup: @@ -309,17 +302,17 @@ class Tracker: if not date: date = Date.get_today_date_str() - new_obj_date = r_serv_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') - new_obj = r_serv_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) + new_obj_date = r_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') + new_obj = r_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) # MATCHES if new_obj: - r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', 1, 'total') - r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', 1, obj_type) + r_tracker.zincrby(f'tracker:match:{self.uuid}', 1, 'total') + r_tracker.zincrby(f'tracker:match:{self.uuid}', 1, obj_type) # Only save date for daterange objects - Needed for the DB Cleaner if obj_type != 'item': # not obj_date: - r_serv_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) - r_serv_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') + r_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) + r_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') if new_obj_date: self.update_daterange(date) @@ -327,13 +320,13 @@ class Tracker: self._add_to_dashboard(obj_type, subtype, obj_id) def get_objs_by_type(self, obj_type): - return r_serv_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}') + return r_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}') def get_nb_objs_by_date(self, date): - return r_serv_tracker.scard(f'tracker:objs:{self.uuid}:{date}') + return r_tracker.scard(f'tracker:objs:{self.uuid}:{date}') def get_objs_by_date(self, date): - return r_serv_tracker.smembers(f'tracker:objs:{self.uuid}:{date}') + return r_tracker.smembers(f'tracker:objs:{self.uuid}:{date}') def get_objs_by_daterange(self, date_from, date_to): objs = set() @@ -345,24 +338,24 @@ class Tracker: if obj_type == 'item': return [item_basic.get_item_date(obj_id)] else: - return r_serv_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}') + return r_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}') def remove(self, obj_type, subtype, obj_id): if not subtype: subtype = '' for date in self.get_obj_dates(obj_type, subtype, obj_id): - r_serv_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') - r_serv_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) + r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') + r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) - r_serv_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) - r_serv_tracker.srem(f'tracker:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') + r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) + r_tracker.srem(f'tracker:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') # MATCHES - r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', -1, 'total') - r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', -1, obj_type) + r_tracker.zincrby(f'tracker:match:{self.uuid}', -1, 'total') + r_tracker.zincrby(f'tracker:match:{self.uuid}', -1, obj_type) self.update_daterange() - # TODO escape tags ???? + # TODO escape custom tags # TODO escape mails ???? def create(self, tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None): if self.exists(): @@ -377,7 +370,7 @@ class Tracker: domain = to_track.split(" ")[0] typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1 for typo in typo_generation: - r_serv_tracker.sadd(f'tracker:typosquatting:{to_track}', typo) + r_tracker.sadd(f'tracker:typosquatting:{to_track}', typo) # create metadata self._set_field('tracked', to_track) @@ -390,11 +383,11 @@ class Tracker: self._set_field('webhook', webhook) # create all tracker set - r_serv_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # TODO RENAME ???? + r_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # create tracker - uuid map - r_serv_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) # TODO RENAME ???? - r_serv_tracker.sadd('trackers:all', self.uuid) - r_serv_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) + r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) + r_tracker.sadd('trackers:all', self.uuid) + r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) # TRACKER LEVEL self._set_level(level, tracker_type=tracker_type, user=user_id) @@ -407,7 +400,6 @@ class Tracker: if mails: self._set_mails(mails) - # TODO Delete filters # Filters if not filters: filters = {} @@ -416,8 +408,8 @@ class Tracker: else: self.set_filters(filters) for obj_type in filters: - r_serv_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) - r_serv_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') + r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) + r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') self._set_field('last_change', time.time()) @@ -451,9 +443,9 @@ class Tracker: if tracker_type != old_type: # LEVEL if old_level == 0: - r_serv_tracker.srem(f'user:tracker:{user_id}:{old_type}', self.uuid) + r_tracker.srem(f'user:tracker:{user_id}:{old_type}', self.uuid) elif old_level == 1: - r_serv_tracker.srem(f'global:tracker:{old_type}', self.uuid) + r_tracker.srem(f'global:tracker:{old_type}', self.uuid) self._set_level(level, tracker_type=tracker_type, user=user_id) # Delete OLD YARA Rule File if old_type == 'yara': @@ -464,21 +456,21 @@ class Tracker: self._set_field('type', tracker_type) # create all tracker set - r_serv_tracker.srem(f'all:tracker:{old_type}', old_to_track) - r_serv_tracker.sadd(f'all:tracker:{tracker_type}', to_track) + r_tracker.srem(f'all:tracker:{old_type}', old_to_track) + r_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # create tracker - uuid map - r_serv_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid) - r_serv_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) + r_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid) + r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) # create all tracker set by type - r_serv_tracker.srem(f'trackers:all:{old_type}', self.uuid) - r_serv_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) + r_tracker.srem(f'trackers:all:{old_type}', self.uuid) + r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) # Same Type elif level != old_level: if level == 0: - r_serv_tracker.srem('global:tracker', self.uuid) + r_tracker.srem('global:tracker', self.uuid) elif level == 1: - r_serv_tracker.srem(f'user:tracker:{user_id}', self.uuid) + r_tracker.srem(f'user:tracker:{user_id}', self.uuid) self._set_level(level, tracker_type=tracker_type, user=user_id) # To Track Edited @@ -489,20 +481,20 @@ class Tracker: self._set_field('webhook', webhook) # Tags - nb_old_tags = r_serv_tracker.scard(f'tracker:tags:{self.uuid}') + nb_old_tags = r_tracker.scard(f'tracker:tags:{self.uuid}') if nb_old_tags > 0 or tags: - r_serv_tracker.delete(f'tracker:tags:{self.uuid}') + r_tracker.delete(f'tracker:tags:{self.uuid}') self._set_tags(tags) # Mails - nb_old_mails = r_serv_tracker.scard(f'tracker:mail:{self.uuid}') + nb_old_mails = r_tracker.scard(f'tracker:mail:{self.uuid}') if nb_old_mails > 0 or mails: - r_serv_tracker.delete(f'tracker:mail:{self.uuid}') + r_tracker.delete(f'tracker:mail:{self.uuid}') self._set_mails(mails) - nb_old_sources = r_serv_tracker.scard(f'tracker:sources:{self.uuid}') # TODO FILTERS + nb_old_sources = r_tracker.scard(f'tracker:sources:{self.uuid}') # TODO FILTERS if nb_old_sources > 0 or sources: - r_serv_tracker.delete(f'tracker:sources:{self.uuid}') + r_tracker.delete(f'tracker:sources:{self.uuid}') self._set_sources(sources) # Refresh Trackers @@ -532,24 +524,24 @@ def get_trackers_types(): return ['word', 'set', 'regex', 'typosquatting', 'yara'] def get_trackers(): - return r_serv_tracker.smembers(f'trackers:all') + return r_tracker.smembers(f'trackers:all') def get_trackers_by_type(tracker_type): - return r_serv_tracker.smembers(f'trackers:all:{tracker_type}') + return r_tracker.smembers(f'trackers:all:{tracker_type}') def _get_tracked_by_obj_type(tracker_type, obj_type): - return r_serv_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}') + return r_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}') def get_trackers_by_tracked_obj_type(tracker_type, obj_type, tracked): trackers_uuid = set() - for res in r_serv_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'): + for res in r_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'): tracker_uuid, tracker_obj_type = res.split(':', 1) if tracker_obj_type == obj_type: trackers_uuid.add(tracker_uuid) return trackers_uuid def get_trackers_by_tracked(tracker_type, tracked): - return r_serv_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}') + return r_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}') def get_user_trackers_by_tracked(tracker_type, tracked, user_id): user_trackers = get_user_trackers(user_id, tracker_type=tracker_type) @@ -557,31 +549,31 @@ def get_user_trackers_by_tracked(tracker_type, tracked, user_id): return trackers_uuid.intersection(user_trackers) def get_trackers_tracked_by_type(tracker_type): - return r_serv_tracker.smembers(f'all:tracker:{tracker_type}') + return r_tracker.smembers(f'all:tracker:{tracker_type}') def get_global_trackers(tracker_type=None): if tracker_type: - return r_serv_tracker.smembers(f'global:tracker:{tracker_type}') + return r_tracker.smembers(f'global:tracker:{tracker_type}') else: - return r_serv_tracker.smembers('global:tracker') + return r_tracker.smembers('global:tracker') def get_user_trackers(user_id, tracker_type=None): if tracker_type: - return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') + return r_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') else: - return r_serv_tracker.smembers(f'user:tracker:{user_id}') + return r_tracker.smembers(f'user:tracker:{user_id}') def get_nb_global_trackers(tracker_type=None): if tracker_type: - return r_serv_tracker.scard(f'global:tracker:{tracker_type}') + return r_tracker.scard(f'global:tracker:{tracker_type}') else: - return r_serv_tracker.scard('global:tracker') + return r_tracker.scard('global:tracker') def get_nb_user_trackers(user_id, tracker_type=None): if tracker_type: - return r_serv_tracker.scard(f'user:tracker:{user_id}:{tracker_type}') + return r_tracker.scard(f'user:tracker:{user_id}:{tracker_type}') else: - return r_serv_tracker.scard(f'user:tracker:{user_id}') + return r_tracker.scard(f'user:tracker:{user_id}') def get_user_trackers_meta(user_id, tracker_type=None): metas = [] @@ -616,7 +608,7 @@ def get_trackers_graph_by_day(l_trackers, num_day=31, date_from=None, date_to=No def get_trackers_dashboard(): trackers = [] - for raw in r_serv_tracker.lrange('trackers:dashboard', 0, -1): + for raw in r_tracker.lrange('trackers:dashboard', 0, -1): tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) tracker = Tracker(tracker_uuid) meta = tracker.get_meta(options={'tags'}) @@ -627,7 +619,7 @@ def get_trackers_dashboard(): def get_user_dashboard(user_id): # TODO SORT + REMOVE OLDER ROWS trackers = [] - for raw in r_serv_tracker.lrange(f'trackers:user:{user_id}', 0, -1): + for raw in r_tracker.lrange(f'trackers:user:{user_id}', 0, -1): tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) tracker = Tracker(tracker_uuid) meta = tracker.get_meta(options={'tags'}) @@ -647,7 +639,6 @@ def get_trackers_stats(user_id): return stats - ## Cache ## # TODO API: Check Tracker type def trigger_trackers_refresh(tracker_type): @@ -660,36 +651,13 @@ def get_tracker_last_updated_by_type(tracker_type): return float(epoch_update) # - Cache - # - - -# Dashboard by user -> tracker - # Add get last tracker in User class - -# Global/User dashboard last trackers - -# -> in ADD function - - - - - - - - - - - - - - - ## Objects ## def is_obj_tracked(obj_type, subtype, obj_id): - return r_serv_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') + return r_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') def get_obj_trackers(obj_type, subtype, obj_id): - return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') + return r_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') def delete_obj_trackers(obj_type, subtype, obj_id): for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id): @@ -718,7 +686,7 @@ def is_tracked_in_user_level(tracked, tracker_type, user_id): def api_check_tracker_uuid(tracker_uuid): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 - if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): + if not r_tracker.exists(f'tracker:{tracker_uuid}'): return {"status": "error", "reason": "Unknown uuid"}, 404 return None @@ -735,7 +703,7 @@ def api_check_tracker_acl(tracker_uuid, user_id): def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 - tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') + tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') if not tracker_creator: return {"status": "error", "reason": "Unknown uuid"}, 404 user = User(user_id) @@ -751,16 +719,16 @@ def fix_tracker_stats_per_day(tracker_uuid): date_from = tracker.get_date() date_to = Date.get_today_date_str() # delete stats - r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}') - r_serv_tracker.hdel(f'tracker:{tracker_uuid}', 'first_seen') - r_serv_tracker.hdel(f'tracker:{tracker_uuid}', 'last_seen') + r_tracker.delete(f'tracker:stat:{tracker_uuid}') + r_tracker.hdel(f'tracker:{tracker_uuid}', 'first_seen') + r_tracker.hdel(f'tracker:{tracker_uuid}', 'last_seen') # create new stats for date_day in Date.substract_date(date_from, date_to): date_day = int(date_day) - nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}') + nb_items = r_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}') if nb_items: - r_serv_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day)) + r_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day)) # update first_seen/last_seen tracker.update_daterange(date_day) @@ -772,41 +740,43 @@ def fix_tracker_item_link(tracker_uuid): if date_from and date_to: for date_day in Date.substract_date(date_from, date_to): - l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') + l_items = r_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') for item_id in l_items: - r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) + r_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) def fix_all_tracker_uuid_list(): - r_serv_tracker.delete(f'trackers:all') + r_tracker.delete(f'trackers:all') for tracker_type in get_trackers_types(): - r_serv_tracker.delete(f'trackers:all:{tracker_type}') + r_tracker.delete(f'trackers:all:{tracker_type}') for tracked in get_trackers_tracked_by_type(tracker_type): l_tracker_uuid = get_trackers_by_tracked(tracker_type, tracked) for tracker_uuid in l_tracker_uuid: - r_serv_tracker.sadd(f'trackers:all', tracker_uuid) - r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) + r_tracker.sadd(f'trackers:all', tracker_uuid) + r_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) ##-- FIX DB --## #### CREATE TRACKER #### -def api_validate_tracker_to_add(to_track , tracker_type, nb_words=1): - if tracker_type=='regex': +def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1): + if tracker_type == 'regex': if not is_valid_regex(to_track): return {"status": "error", "reason": "Invalid regex"}, 400 - elif tracker_type=='word' or tracker_type=='set': + elif tracker_type == 'word' or tracker_type == 'set': # force lowercase to_track = to_track.lower() word_set = set(to_track) set_inter = word_set.intersection(special_characters) if set_inter: - return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400 + return {"status": "error", + "reason": f'special character(s) not allowed: {set_inter}', + "message": "Please use a python regex or remove all special characters"}, 400 words = to_track.split() # not a word - if tracker_type=='word' and len(words)>1: + if tracker_type == 'word' and len(words) > 1: tracker_type = 'set' # output format: tracker1,tracker2,tracker3;2 - if tracker_type=='set': + if tracker_type == 'set': try: nb_words = int(nb_words) except TypeError: @@ -827,13 +797,13 @@ def api_validate_tracker_to_add(to_track , tracker_type, nb_words=1): domain = to_track.split(" ") if len(domain) > 1: return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400 - if not "." in to_track: + if "." not in to_track: return {"status": "error", "reason": "Invalid domain name"}, 400 - elif tracker_type=='yara_custom': + elif tracker_type == 'yara_custom': if not is_valid_yara_rule(to_track): return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 - elif tracker_type=='yara_default': + elif tracker_type == 'yara_default': if not is_valid_default_yara_rule(to_track): return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: @@ -935,17 +905,11 @@ def api_delete_tracker(data, user_id): tracker = Tracker(tracker_uuid) return tracker.delete(), 200 - - - -##-- CREATE TRACKER --## +## -- CREATE TRACKER -- ## #################### #### WORD - SET #### -def get_words_tracked_list(): # TODO REMOVE ME ???? - return list(r_serv_tracker.smembers('all:tracker:word')) - def get_tracked_words(): to_track = {} for obj_type in get_objects_tracked(): @@ -990,7 +954,7 @@ def get_tracked_regexs(): #### TYPO SQUATTING #### def get_tracked_typosquatting_domains(tracked): - return r_serv_tracker.smembers(f'tracker:typosquatting:{tracked}') + return r_tracker.smembers(f'tracker:typosquatting:{tracked}') def get_tracked_typosquatting(): to_track = {} @@ -1031,7 +995,7 @@ def get_all_default_yara_rules_by_type(yara_types): return [] def get_all_tracked_yara_files(filter_disabled=False): - yara_files = r_serv_tracker.smembers('all:tracker:yara') + yara_files = r_tracker.smembers('all:tracker:yara') if not yara_files: yara_files = [] if filter_disabled: @@ -1104,7 +1068,7 @@ def is_valid_default_yara_rule(yara_rule, verbose=True): def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None): if yara_rule_type == 'yara_custom': - if not tracker_uuid: + if not tracker_uuid: tracker_uuid = str(uuid.uuid4()) filename = os.path.join('custom-rules', tracker_uuid + '.yar') with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f: @@ -1131,7 +1095,7 @@ def get_yara_rule_content(yara_rule): # incorrect filename if not os.path.commonprefix([filename, yara_dir]) == yara_dir: - return '' # # TODO: throw exception + return '' # # TODO: throw exception with open(filename, 'r') as f: rule_content = f.read() @@ -1170,91 +1134,87 @@ def get_yara_rule_content_restapi(request_dict): rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8') return {'status': 'success', 'content': rule_content}, 200 - - -##-- YARA --## +## -- YARA -- ## ###################### #### RETRO - HUNT #### # state: pending/running/completed/paused -# task keys: -## tracker:retro_hunt:task:{task_uuid} state -# start_time -# end_time -# date_from -# date_to -# creator -# timeout -# date -# type - class RetroHunt: def __init__(self, task_uuid): self.uuid = task_uuid def exists(self): - return r_serv_tracker.exists(f'tracker:retro_hunt:task:{self.uuid}') + return r_tracker.exists(f'retro_hunt:{self.uuid}') + + def _get_field(self, field): + return r_tracker.hget(f'retro_hunt:{self.uuid}', field) def _set_field(self, field, value): - return r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', field, value) + return r_tracker.hset(f'retro_hunt:{self.uuid}', field, value) def get_creator(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'creator') + return self._get_field('creator') def get_date(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date') - - def get_date_from(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date_from') - - def get_date_to(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date_to') + return self._get_field('date') def get_last_analyzed(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'last') + return self._get_field('last') + + def set_last_analyzed(self, obj_type, subtype, obj_id): + return self._set_field('last', f'{obj_type}:{subtype}:{obj_id}') + + def get_last_analyzed_cache(self): + r_cache.hget(f'retro_hunt:task:{self.uuid}', 'obj') + + def set_last_analyzed_cache(self, obj_type, subtype, obj_id): + r_cache.hset(f'retro_hunt:task:{self.uuid}', 'obj', f'{obj_type}:{subtype}:{obj_id}') def get_name(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'name') + return self._get_field('name') def get_description(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'description') + return self._get_field('description') def get_timeout(self): - res = r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'timeout') + res = self._get_field('timeout') if res: return int(res) else: return 30 # # TODO: FIXME use instance limit - def get_sources(self, r_sort=False): # TODO ADAPT TO ALL OBJECTS ??? - sources = r_serv_tracker.smembers(f'tracker:retro_hunt:task:sources:{self.uuid}') - if not sources: - sources = set(item_basic.get_all_items_sources(filter_dir=False)) - if r_sort: - sources = sorted(sources) - return sources + def get_filters(self): + filters = self._get_field('filters') + if not filters: + return {} + else: + return json.loads(filters) + + def set_filters(self, filters): + if filters: + self._set_field('filters', json.dumps(filters)) def get_tags(self): - return r_serv_tracker.smembers(f'tracker:retro_hunt:task:tags:{self.uuid}') + return r_tracker.smembers(f'retro_hunt:tags:{self.uuid}') def get_mails(self): - return r_serv_tracker.smembers(f'tracker:retro_hunt:task:mails:{self.uuid}') + return r_tracker.smembers(f'retro_hunt:mails:{self.uuid}') def get_state(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'state') + return self._get_field('state') def _set_state(self, new_state): curr_state = self.get_state() if curr_state: - r_serv_tracker.srem(f'tracker:retro_hunt:task:{curr_state}', self.uuid) - r_serv_tracker.sadd(f'tracker:retro_hunt:task:{new_state}', self.uuid) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', 'state', new_state) + r_tracker.srem(f'retro_hunt:task:{curr_state}', self.uuid) + r_tracker.sadd(f'retro_hunts:{new_state}', self.uuid) + self._set_field('state', new_state) def get_rule(self, r_compile=False): - rule = r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'rule') + rule = self._get_field('rule') if r_compile: rule = os.path.join(get_yara_rules_dir(), rule) rule_dict = {self.uuid: os.path.join(get_yara_rules_dir(), rule)} @@ -1264,8 +1224,6 @@ class RetroHunt: # add timeout ? def get_meta(self, options=set()): meta = {'uuid': self.uuid, - 'date_from': self.get_date_from(), - 'date_to': self.get_date_to(), 'name': self.get_name(), 'state': self.get_state(), 'rule': self.get_rule(), @@ -1273,23 +1231,28 @@ class RetroHunt: if 'creator' in options: meta['creator'] = self.get_creator() if 'date' in options: - meta['date'] = self.get_date() + meta['date'] = self.get_date() if 'description' in options: meta['description'] = self.get_description() if 'mails' in options: meta['mails'] = self.get_mails() if 'nb_match' in options: meta['nb_match'] = self.get_nb_match() + if 'nb_objs' in options: + meta['nb_objs'] = self.get_nb_objs() if 'progress' in options: meta['progress'] = self.get_progress() - if 'sources' in options: - meta['progress'] = self.get_sources(r_sort=True) + if 'filters' in options: + meta['filters'] = self.get_filters() if 'tags' in options: meta['tags'] = self.get_tags() return meta + def is_paused(self): + return r_tracker.sismember('retro_hunts:paused', self.uuid) + def to_pause(self): - to_pause = r_cache.hget(f'tracker:retro_hunt:task:{self.uuid}', 'pause') + to_pause = r_cache.hget(f'retro_hunt:{self.uuid}', 'pause') if to_pause: return True else: @@ -1297,140 +1260,166 @@ class RetroHunt: def pause(self): self._set_state('paused') - r_cache.hset(f'tracker:retro_hunt:task:{self.uuid}', 'pause', time.time()) + r_cache.hset(f'retro_hunt:{self.uuid}', 'pause', time.time()) self.clear_cache() def resume(self): - r_cache.hdel(f'tracker:retro_hunt:task:{self.uuid}', 'pause') + r_cache.hdel(f'retro_hunt:{self.uuid}', 'pause') self._set_state('pending') + def is_running(self): + return r_tracker.sismember('retro_hunts:running', self.uuid) + def run(self): # TODO ADD MORE CHECK self._set_state('running') def complete(self): self._set_state('completed') - self.update_nb_match() self.clear_cache() + r_tracker.hdel(f'retro_hunt:{self.uuid}', 'last') def get_progress(self): if self.get_state() == 'completed': progress = 100 else: - progress = r_cache.hget(f'tracker:retro_hunt:task:{self.uuid}', 'progress') + progress = r_cache.hget(f'retro_hunt:{self.uuid}', 'progress') if not progress: - progress = self.compute_progress() + progress = self._get_field('progress') return progress - def compute_progress(self, date_from=None, date_to=None, sources=[], curr_date=None, nb_src_done=0): - # get nb days - if not date_from: - date_from = self.get_date_from() - if not date_to: - date_to = self.get_date_to() - nb_days = Date.get_nb_days_by_daterange(date_from, date_to) - - # nb days completed - if not curr_date: - curr_date = get_retro_hunt_task_current_date(task_uuid) #################################################### - nb_days_done = Date.get_nb_days_by_daterange(date_from, curr_date) - 1 - - # sources - if not sources: - nb_sources = len(self.get_sources()) - else: - nb_sources = len(sources) - - # get progress - progress = ((nb_days_done * nb_sources) + nb_src_done) * 100 / (nb_days * nb_sources) - return int(progress) - - # # TODO: # FIXME: # Cache - def set_progress(self, progress): - r_cache.hset(f'tracker:retro_hunt:task:{self.uuid}', 'progress', progress) + res = r_cache.hset(f'retro_hunt:{self.uuid}', 'progress', progress) + if res: + self._set_field('progress', progress) def get_nb_match(self): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'nb_match') + return self._get_field('nb_match') + + def _incr_nb_match(self): + r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', 1) + + def _decr_nb_match(self): + r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', -1) def _set_nb_match(self, nb_match): - r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', 'nb_match', nb_match) - - def update_nb_match(self): - l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{self.uuid}', 0, -1, withscores=True) - nb_match = 0 - for row in l_date_value: - nb_match += int(row[1]) - self._set_nb_match(nb_match) + self._set_field('nb_match', nb_match) def clear_cache(self): - r_cache.delete(f'tracker:retro_hunt:task:{self.uuid}') + r_cache.delete(f'retro_hunt:{self.uuid}') - def create(self, name, rule, date_from, date_to, creator, description=None, mails=[], tags=[], timeout=30, sources=[], state='pending'): + def get_nb_objs_by_type(self, obj_type): + return r_tracker.scard(f'retro_hunt:objs:{self.uuid}:{obj_type}') + + def get_objs_by_type(self, obj_type): + return r_tracker.smembers(f'retro_hunt:objs:{self.uuid}:{obj_type}') + + def get_nb_objs(self): + objs = {} + for obj_type in get_objects_retro_hunted(): + objs[obj_type] = self.get_nb_objs_by_type(obj_type) + return objs + + def get_objs(self): + objs = [] + for obj_type in get_objects_retro_hunted(): + for obj in self.get_objs_by_type(obj_type): + subtype, obj_id = obj.split(':', 1) + objs.append((obj_type, subtype, obj_id)) + return objs + + def add(self, obj_type, subtype, obj_id): + # match by object type: + r_tracker.sadd(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') + # MAP object -> retro hunt + r_tracker.sadd(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid) + self._incr_nb_match() + + def remove(self, obj_type, subtype, obj_id): + # match by object type: + r_tracker.srem(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') + # MAP object -> retro hunt + r_tracker.srem(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid) + self._decr_nb_match() + + def create(self, name, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending'): if self.exists(): raise Exception('Error: Retro Hunt Task already exists') self._set_field('name', escape(name)) - self._set_field('rule', rule) # TODO FORMAT ??? + self._set_field('rule', rule) self._set_field('date', datetime.date.today().strftime("%Y%m%d")) self._set_field('name', escape(name)) - self._set_field('date_from', date_from) - self._set_field('date_to', date_to) self._set_field('creator', creator) if description: self._set_field('description', description) if timeout: self._set_field('timeout', int(timeout)) - for source in sources: - r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{self.uuid}', escape(source)) for tag in tags: - tag = escape(tag) - r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{self.uuid}', tag) + # tag = escape(tag) + r_tracker.sadd(f'retro_hunt:tags:{self.uuid}', tag) Tag.create_custom_tag(tag) for mail in mails: - r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{self.uuid}', escape(mail)) + r_tracker.sadd(f'retro_hunt:mails:{self.uuid}', escape(mail)) - r_serv_tracker.sadd('tracker:retro_hunt:task:all', self.uuid) + # TODO Delete filters - SAVE DEFAULT OBJECTS ??? + # Filters + # if not filters: + # filters = {} + # for obj_type in get_objects_tracked(): + # filters[obj_type] = {} + if filters: + self.set_filters(filters) + + r_tracker.sadd('retro_hunts:all', self.uuid) # add to pending tasks if state not in ('pending', 'completed', 'paused'): state = 'pending' self._set_state(state) - - # TODO Delete Rule + # TODO Delete Rule custom def delete(self): - if r_serv_tracker.sismember('tracker:retro_hunt:task:running', self.uuid): + if self.is_running(): return None - r_serv_tracker.srem('tracker:retro_hunt:task:pending', self.uuid) - r_serv_tracker.delete(f'tracker:retro_hunt:task:{self.uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:sources:{self.uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:tags:{self.uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:mails:{self.uuid}') + r_tracker.srem('retro_hunts:pending', self.uuid) + r_tracker.delete(f'retro_hunts:{self.uuid}') + r_tracker.delete(f'retro_hunt:tags:{self.uuid}') + r_tracker.delete(f'retro_hunt:mails:{self.uuid}') - for item_date in get_retro_hunt_all_item_dates(task_uuid): ############################ TODO OBJ ####################### - r_serv_tracker.delete(f'tracker:retro_hunt:task:item:{self.uuid}:{item_date}') + for obj in self.get_objs(): + self.remove(obj[0], obj[1], obj[2]) - r_serv_tracker.srem('tracker:retro_hunt:task:all', self.uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:pending', self.uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:paused', self.uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:completed', self.uuid) + r_tracker.srem('retro_hunts:all', self.uuid) + r_tracker.srem('retro_hunts:pending', self.uuid) + r_tracker.srem('retro_hunts:paused', self.uuid) + r_tracker.srem('retro_hunts:completed', self.uuid) self.clear_cache() return self.uuid -def create_retro_hunt(name, rule_type, rule, date_from, date_to, creator, description=None, mails=[], tags=[], timeout=30, sources=[], state='pending', task_uuid=None): +def create_retro_hunt(name, rule_type, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending', task_uuid=None): if not task_uuid: task_uuid = str(uuid.uuid4()) retro_hunt = RetroHunt(task_uuid) # rule_type: yara_default - yara custom rule = save_yara_rule(rule_type, rule, tracker_uuid=retro_hunt.uuid) - retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, - timeout=timeout, sources=sources, state=state) + retro_hunt.create(name, rule, creator, description=description, mails=mails, tags=tags, + timeout=timeout, filters=filters, state=state) return retro_hunt.uuid +# TODO +# def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None): +# retro_hunt = RetroHunt(task_uuid) +# retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, +# timeout=timeout, sources=sources, state=state) +# if last_id: +# set_retro_hunt_last_analyzed(task_uuid, last_id) +# retro_hunt._set_nb_match(nb_match) +# retro_hunt._set_field('date', date) + ## ? ? ? # set tags # set mails @@ -1439,24 +1428,24 @@ def create_retro_hunt(name, rule_type, rule, date_from, date_to, creator, descri # SET Retro Hunts def get_all_retro_hunt_tasks(): - return r_serv_tracker.smembers('tracker:retro_hunt:task:all') + return r_tracker.smembers('retro_hunts:all') def get_retro_hunt_pending_tasks(): - return r_serv_tracker.smembers('tracker:retro_hunt:task:pending') + return r_tracker.smembers('retro_hunts:pending') def get_retro_hunt_running_tasks(): - return r_serv_tracker.smembers('tracker:retro_hunt:task:running') + return r_tracker.smembers('retro_hunts:running') def get_retro_hunt_paused_tasks(): - return r_serv_tracker.smembers('tracker:retro_hunt:task:paused') + return r_tracker.smembers('retro_hunts:paused') def get_retro_hunt_completed_tasks(): - return r_serv_tracker.smembers('tracker:retro_hunt:task:completed') + return r_tracker.smembers('retro_hunts:completed') ## Change STATES ## def get_retro_hunt_task_to_start(): - task_uuid = r_serv_tracker.spop('tracker:retro_hunt:task:pending') + task_uuid = r_tracker.spop('retro_hunts:pending') if task_uuid: retro_hunt = RetroHunt(task_uuid) retro_hunt.run() @@ -1464,226 +1453,22 @@ def get_retro_hunt_task_to_start(): ## Metadata ## -def get_retro_hunt_tasks_metas(): +def get_retro_hunt_metas(): tasks = [] for task_uuid in get_all_retro_hunt_tasks(): retro_hunt = RetroHunt(task_uuid) tasks.append(retro_hunt.get_meta(options={'date', 'progress', 'nb_match', 'tags'})) return tasks - - - - - - -def get_retro_hunt_last_analyzed(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'last') - -# Keep history to relaunch on error/pause -def set_retro_hunt_last_analyzed(task_uuid, last_id): - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'last', last_id) - -#################################################################################### -#################################################################################### -#################################################################################### -#################################################################################### - -def set_cache_retro_hunt_task_id(task_uuid, id): - r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'id', id) - -# Others - -# date -# type -# tags -# mails -# name -# description - -# state error - -# TODO -def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None): - retro_hunt = RetroHunt(task_uuid) - retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, - timeout=timeout, sources=sources, state=state) - # TODO - if last_id: - set_retro_hunt_last_analyzed(task_uuid, last_id) - retro_hunt._set_nb_match(nb_match) - retro_hunt._set_field('date', date) - -def get_retro_hunt_task_current_date(task_uuid): - retro_hunt = RetroHunt(task_uuid) - last = get_retro_hunt_last_analyzed(task_uuid) - if last: - curr_date = item_basic.get_item_date(last) - else: - curr_date = retro_hunt.get_date_from() - return curr_date - -def get_retro_hunt_task_nb_src_done(task_uuid, sources=[]): - retro_hunt = RetroHunt(task_uuid) - if not sources: - sources = list(retro_hunt.get_sources(r_sort=True)) - else: - sources = list(sources) - last_id = get_retro_hunt_last_analyzed(task_uuid) - if last_id: - last_source = item_basic.get_source(last_id) - try: - nb_src_done = sources.index(last_source) - except ValueError: - nb_src_done = 0 - else: - nb_src_done = 0 - return nb_src_done - -def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, sources=[]): - retro_hunt = RetroHunt(task_uuid) - if not sources: - sources = retro_hunt.get_sources(r_sort=True) - - # filter last - if filter_last: - last = get_retro_hunt_last_analyzed(task_uuid) - if last: - curr_source = item_basic.get_source(last) - # remove processed sources - set_sources = sources.copy() - for source in sources: - if source != curr_source: - set_sources.remove(source) - else: - break - sources = set_sources - - # return all dirs by day - date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}' - dirs = set() - for source in sources: - dirs.add(os.path.join(source, date)) - return dirs - -# # TODO: move me -def get_items_to_analyze(dir, last=None): - if items_dir == 'PASTES': - full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) - else: - full_dir = os.path.join(items_dir, dir) - if os.path.isdir(full_dir): - all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))]) - # remove processed items - if last: - items_set = all_items.copy() - for item in all_items: - if item != last: - items_set.remove(item) - else: - break - all_items = items_set - return all_items - else: - return [] - -# # TODO: ADD MAP ID => Retro_Hunt -def save_retro_hunt_match(task_uuid, id, object_type='item'): - item_date = item_basic.get_item_date(id) - res = r_serv_tracker.sadd(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}', id) - # track nb item by date - if res == 1: - r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', 1, int(item_date)) - # Add map obj_id -> task_uuid - r_serv_tracker.sadd(f'obj:retro_hunt:item:{id}', task_uuid) - -def delete_retro_hunt_obj(task_uuid, obj_type, obj_id): - item_date = item_basic.get_item_date(obj_id) - res = r_serv_tracker.srem(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}', obj_id) - get_retro_hunt_nb_item_by_day() - # track nb item by date - if res == 1: - r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', -1, int(item_date)) - # Add map obj_id -> task_uuid - r_serv_tracker.srem(f'obj:retro_hunt:item:{obj_id}', task_uuid) - -# TODO -def delete_object_reto_hunts(obj_type, obj_id): - pass -# # get items all retro hunts -# for task_uuid in : ############################################# -# delete_retro_hunt_obj(task_uuid, obj_type, obj_id) - -def get_retro_hunt_all_item_dates(task_uuid): - return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1) - -def get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to): - all_item_id = set() - if date_from and date_to: - l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) - if l_date_match: - dict_date_match = dict(l_date_match) - for date_day in Date.substract_date(date_from, date_to): - if date_day in dict_date_match: - all_item_id |= r_serv_tracker.smembers(f'tracker:retro_hunt:task:item:{task_uuid}:{date_day}') - return all_item_id - -def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None): - list_stats = [] - for task_uuid in l_task_uuid: - dict_task_data = [] - retro_hunt = RetroHunt(task_uuid) - - l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) ######################## - if l_date_match: - dict_date_match = dict(l_date_match) - if not date_from: - date_from = min(dict_date_match) - if not date_to: - date_to = max(dict_date_match) - - date_range = Date.substract_date(date_from, date_to) - for date_day in date_range: - nb_seen_this_day = int(dict_date_match.get(date_day, 0)) - dict_task_data.append({"date": date_day,"value": int(nb_seen_this_day)}) - list_stats.append({"name": retro_hunt.get_name(),"Data": dict_task_data}) - return list_stats - ## API ## def api_check_retro_hunt_task_uuid(task_uuid): if not is_valid_uuid_v4(task_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 - if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'): + retro_hunt = RetroHunt(task_uuid) + if not retro_hunt.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 return None -def api_get_retro_hunt_items(dict_input): - task_uuid = dict_input.get('uuid', None) - res = api_check_retro_hunt_task_uuid(task_uuid) - if res: - return res - - retro_hunt = RetroHunt(task_uuid) - - # TODO SANITIZE DATES - date_from = dict_input.get('date_from', None) - date_to = dict_input.get('date_to', None) - if date_from is None: - date_from = retro_hunt.get_date_from() - if date_to is None: - date_to = date_from - if date_from > date_to: - date_from = date_to - - all_items_id = get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to) - all_items_id = item_basic.get_all_items_metadata_dict(all_items_id) - - res_dict = {'uuid': task_uuid, - 'date_from': date_from, - 'date_to': date_to, - 'items': all_items_id} - return res_dict, 200 - def api_pause_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: @@ -1700,16 +1485,17 @@ def api_resume_retro_hunt_task(task_uuid): if res: return res retro_hunt = RetroHunt(task_uuid) - if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid): - return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400 + if not retro_hunt.is_paused(): + return {"status": "error", + "reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400 retro_hunt.resume() return task_uuid, 200 def api_validate_rule_to_add(rule, rule_type): - if rule_type=='yara_custom': + if rule_type == 'yara_custom': if not is_valid_yara_rule(rule): return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 - elif rule_type=='yara_default': + elif rule_type == 'yara_default': if not is_valid_default_yara_rule(rule): return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: @@ -1737,53 +1523,74 @@ def api_create_retro_hunt_task(dict_input, creator): description = escape(description) description = description[:1000] - res = api_validate_rule_to_add(rule , task_type) - if res[1]!=200: + res = api_validate_rule_to_add(rule, task_type) + if res[1] != 200: return res - tags = dict_input.get('tags', []) - mails = dict_input.get('mails', []) + tags = dict_input.get('tags', []) # TODO escape custom tags + mails = dict_input.get('mails', []) # TODO escape mails res = verify_mail_list(mails) if res: return res - sources = dict_input.get('sources', []) - res = item_basic.verify_sources_list(sources) - if res: - return res + # Filters # TODO MOVE ME + filters = dict_input.get('filters', {}) + if filters: + if filters.keys() == get_objects_retro_hunted(): + filters = {} + for obj_type in filters: + if obj_type not in get_objects_retro_hunted(): + return {"status": "error", "reason": "Invalid Tracker Object type"}, 400 - date_from = dict_input.get('date_from', '') - date_to = dict_input.get('date_to', '') - res = Date.api_validate_str_date_range(date_from, date_to) - if res: - return res + for filter_name in filters[obj_type]: + if filter_name not in {'date_from', 'date_to', 'mimetypes', 'sources', 'subtypes'}: + return {"status": "error", "reason": "Invalid Filter"}, 400 + elif filter_name == 'date_from': + if not Date.validate_str_date(filters[obj_type]['date_from']): + return {"status": "error", "reason": "Invalid date_from"}, 400 + elif filter_name == 'date_to': + if not Date.validate_str_date(filters[obj_type]['date_from']): + return {"status": "error", "reason": "Invalid date_to"}, 400 + elif filter_name == 'mimetypes': # TODO sanityze mimetypes + pass + elif filter_name == 'sources': + if obj_type == 'item': + res = item_basic.verify_sources_list(filters['item']['sources']) + if res: + return res + else: + return {"status": "error", "reason": "Invalid Filter sources"}, 400 + elif filter_name == 'subtypes': + obj_subtypes = set(get_object_all_subtypes(obj_type)) + for subtype in filters[obj_type]['subtypes']: + if subtype not in obj_subtypes: + return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400 - task_uuid = create_retro_hunt(name, task_type, rule, date_from, date_to, creator, description=description, - mails=mails, tags=tags, timeout=30, sources=sources) + if 'date_from' and 'date_to' in filters: + res = Date.api_validate_str_date_range(filters[obj_type]['date_from'], filters[obj_type]['date_to']) + if res: + return res + + task_uuid = create_retro_hunt(name, task_type, rule, creator, description=description, + mails=mails, tags=tags, timeout=30, filters=filters) return {'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200 def api_delete_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res - if r_serv_tracker.sismember('tracker:retro_hunt:task:running', task_uuid): + retro_hunt = RetroHunt(task_uuid) + if retro_hunt.is_running(): return {"status": "error", "reason": "You can't delete a running task"}, 400 else: - retro_hunt = RetroHunt(task_uuid) return retro_hunt.delete(), 200 -#### DB FIX #### -def get_trackers_tags(): - tags = set() - for tracker_uuid in get_trackers(): - tracker = Tracker(tracker_uuid) - for tag in tracker.get_tags(): - tags.add(tag) - for task_uuid in get_all_retro_hunt_tasks(): - retro_hunt = RetroHunt(task_uuid) - for tag in retro_hunt.get_tags(): - tags.add(tag) - return tags +################################################################################ +################################################################################ +################################################################################ +################################################################################ + +#### DB FIX #### TODO def _fix_db_custom_tags(): for tag in get_trackers_tags(): @@ -1792,6 +1599,7 @@ def _fix_db_custom_tags(): #### -- #### + if __name__ == '__main__': _fix_db_custom_tags() diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 4d8aef9f..0f2d9080 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -43,6 +43,9 @@ def get_object_all_subtypes(obj_type): def get_objects_tracked(): return ['decoded', 'item', 'pgp'] +def get_objects_retro_hunted(): + return ['decoded', 'item'] + def get_all_objects_with_subtypes_tuple(): str_objs = [] for obj_type in get_all_objects(): @@ -56,6 +59,21 @@ def get_all_objects_with_subtypes_tuple(): ##-- AIL OBJECTS --## +#### Redis #### + +def _parse_zscan(response): + cursor, r = response + it = iter(r) + return str(cursor), list(it) + +def zscan_iter(r_redis, name): # count ??? + cursor = 0 + while cursor != "0": + cursor, data = _parse_zscan(r_redis.zscan(name, cursor=cursor)) + yield from data + +## -- Redis -- ## + def paginate_iterator(iter_elems, nb_obj=50, page=1): dict_page = {'nb_all_elem': len(iter_elems)} nb_pages = dict_page['nb_all_elem'] / nb_obj diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index f1846da8..39c8443f 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -372,6 +372,67 @@ def search_decodeds_by_name(name_to_search, r_pos=False): decodeds[decoded_name]['hl-end'] = res.end() return decodeds + +############################################################################ + +def get_decodeds_dir(): + decodeds_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR) + if not decodeds_dir.endswith("/"): + decodeds_dir = f"{decodeds_dir}/" + return decodeds_dir + +# Generator + +def get_nb_decodeds_objects(filters={}): + nb = 0 + if 'mimetypes' in filters: + mimetypes = filters['mimetypes'] + else: + mimetypes = get_all_mimetypes() + d_dir = get_decodeds_dir() + for mimetype in mimetypes: + for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)): + nb += len(files) + return nb + +def get_all_decodeds_objects(filters={}): + if 'mimetypes' in filters: + # TODO sanityze mimetype + mimetypes = filters['mimetypes'] + else: + mimetypes = get_all_mimetypes() + mimetypes = sorted(mimetypes) + + if filters.get('start'): + _, start_id = filters['start'].split(':', 1) + decoded = Decoded(start_id) + # remove sources + start_mimetype = decoded.get_mimetype() + i = 0 + while start_mimetype and len(mimetypes) > i: + if mimetypes[i] == start_mimetype: + mimetypes = mimetypes[i:] + start_mimetype = None + i += 1 + else: + start_id = None + + d_dir = get_decodeds_dir() + for mimetype in mimetypes: + for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)): + if start_id: + i = 0 + while start_id and len(files) > i: + if files[i] == start_id: + files = files[i:] + start_id = None + i += 1 + if i >= len(files): + files = [] + for file in files: + yield Decoded(file).id + + ############################################################################ def sanityze_decoder_names(decoder_name): @@ -538,6 +599,12 @@ def get_all_decodeds_files(): return decodeds -# if __name__ == '__main__': +if __name__ == '__main__': # name_to_search = '4d36' -# print(search_decodeds_by_name(name_to_search)) \ No newline at end of file +# print(search_decodeds_by_name(name_to_search)) +# filters = {'mimetypes': ['text/html']} + filters = {'start': ':1a005f82a4ae0940205c8fd81fd14838845696be'} + # filters = {} + gen = get_all_decodeds_objects(filters=filters) + for f in gen: + print(f) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index f107ca6c..2175b3f0 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -22,7 +22,8 @@ from lib.ail_core import get_ail_uuid from lib.objects.abstract_object import AbstractObject from lib.ConfigLoader import ConfigLoader from lib import item_basic -from lib.data_retention_engine import update_obj_date +from lib.data_retention_engine import update_obj_date, get_obj_date_first +from packages import Date from flask import url_for @@ -406,6 +407,115 @@ def _manual_set_items_date_first_last(): if last != 0: update_obj_date(last, 'item') +################################################################################ +################################################################################ +################################################################################ + +def get_nb_items_objects(filters={}): + nb = 0 + date_from = filters.get('date_from') + date_to = filters.get('date_to') + if 'sources' in filters: + sources = filters['sources'] + else: + sources = get_all_sources() + sources = sorted(sources) + + # date + if date_from and date_to: + daterange = Date.get_daterange(date_from, date_to) + elif date_from: + daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + elif date_to: + date_from = get_obj_date_first('item') + daterange = Date.get_daterange(date_from, date_to) + else: + date_from = get_obj_date_first('item') + daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + + for source in sources: + for date in daterange: + date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}' + full_dir = os.path.join(ITEMS_FOLDER, source, date) + if not os.path.isdir(full_dir): + continue + nb += len(os.listdir(full_dir)) + return nb + +def get_all_items_objects(filters={}): + date_from = filters.get('date_from') + date_to = filters.get('date_to') + if 'sources' in filters: + sources = filters['sources'] + else: + sources = get_all_sources() + sources = sorted(sources) + if filters.get('start'): + _, start_id = filters['start'].split(':', 1) + item = Item(start_id) + # remove sources + start_source = item.get_source() + i = 0 + while start_source and len(sources) > i: + if sources[i] == start_source: + sources = sources[i:] + start_source = None + i += 1 + start_date = item.get_date() + else: + start_id = None + start_date = None + + # date + if date_from and date_to: + daterange = Date.get_daterange(date_from, date_to) + elif date_from: + daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + elif date_to: + date_from = get_obj_date_first('item') + daterange = Date.get_daterange(date_from, date_to) + else: + date_from = get_obj_date_first('item') + daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + if start_date: + if int(start_date) > int(date_from): + i = 0 + while start_date and len(daterange) > i: + if daterange[i] == start_date: + daterange = daterange[i:] + start_date = None + i += 1 + + for source in sources: + for date in daterange: + date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}' + full_dir = os.path.join(ITEMS_FOLDER, source, date) + s_dir = os.path.join(source, date) + if not os.path.isdir(full_dir): + continue + + # TODO replace by os.scandir() ???? + all_items = sorted([os.path.join(s_dir, f) + for f in os.listdir(full_dir) + if os.path.isfile(os.path.join(full_dir, f))]) + # start obj id + if start_id: + i = 0 + while start_id and len(all_items) > i: + if all_items[i] == start_id: + if i == len(all_items): + all_items = [] + else: + all_items = all_items[i+1:] + start_id = None + i += 1 + for obj_id in all_items: + yield Item(obj_id) + +################################################################################ +################################################################################ +################################################################################ + #### API #### def api_get_item(data): @@ -810,9 +920,13 @@ def create_item(obj_id, obj_metadata, io_content): # delete_item(child_id) -# if __name__ == '__main__': +if __name__ == '__main__': # content = 'test file content' # duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]} # # item = Item('tests/2020/01/02/test_save.gz') # item.create(content, _save=False) + filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'} + gen = get_all_items_objects(filters=filters) + for obj_id in gen: + print(obj_id.id) diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index fd7297b9..0560c231 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id, get_all_id_iterator config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -128,8 +128,22 @@ def search_pgps_by_name(name_to_search, subtype, r_pos=False): pgps[pgp_name]['hl-end'] = res.end() return pgps +def get_all_pgps_objects(filters={}): + if 'subtypes' in filters: + subtypes = filters['subtypes'] + else: + subtypes = get_all_subtypes() + for subtype in subtypes: + for z_tuple in get_all_id_iterator('pgp', subtype): + obj_id, _ = z_tuple + yield Pgp(obj_id, subtype) -# if __name__ == '__main__': + +if __name__ == '__main__': # name_to_search = 'ex' # subtype = 'name' # print(search_pgps_by_name(name_to_search, subtype)) + gen = get_all_pgps_objects(filters={'subtypes': ['key']}) + for f in gen: + print(f) + diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 690c8b81..35f620e1 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects.abstract_object import AbstractObject -from lib.ail_core import get_object_all_subtypes +from lib.ail_core import get_object_all_subtypes, zscan_iter from lib.ConfigLoader import ConfigLoader from lib.item_basic import is_crawled, get_item_domain from lib.data_retention_engine import update_obj_date @@ -176,6 +176,9 @@ class AbstractSubtypeObject(AbstractObject, ABC): def get_all_id(obj_type, subtype): return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1) +def get_all_id_iterator(obj_type, subtype): + return zscan_iter(r_object, f'{obj_type}_all:{subtype}') + def get_subtypes_objs_by_date(obj_type, subtype, date): return r_object.hkeys(f'{obj_type}:{subtype}:{date}') diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 0a1e2f2c..01445c9e 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -9,16 +9,16 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple +from lib.ail_core import get_all_objects, get_object_all_subtypes from lib import correlations_engine from lib import btc_ail from lib import Tag from lib.objects import CryptoCurrencies from lib.objects.Cves import Cve -from lib.objects.Decodeds import Decoded +from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects from lib.objects.Domains import Domain -from lib.objects.Items import Item +from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Usernames @@ -114,6 +114,9 @@ def get_obj_tags(obj_type, subtype, id): obj = get_object(obj_type, subtype, id) return obj.get_tags() +def is_obj_tags_safe(obj_type, subtype, id): + obj = get_object(obj_type, subtype, id) + return obj.is_tags_safe() def add_obj_tag(obj_type, subtype, id, tag): obj = get_object(obj_type, subtype, id) @@ -143,6 +146,10 @@ def get_objects_meta(objs, options=set(), flask_context=False): obj_type = obj['type'] subtype = obj['subtype'] obj_id = obj['id'] + elif isinstance(obj, tuple): + obj_type = obj[0] + subtype = obj[1] + obj_id = obj[2] else: obj_type, subtype, obj_id = obj.split(':', 2) metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context)) @@ -184,6 +191,27 @@ def is_filtered(obj, filters): return True return False +def obj_iterator(obj_type, filters): + if obj_type == 'decoded': + return get_all_decodeds_objects(filters=filters) + elif obj_type == 'item': + return get_all_items_objects(filters=filters) + elif obj_type == 'pgp': + return Pgps.get_all_pgps_objects(filters=filters) + +def card_objs_iterators(filters): + nb = 0 + for obj_type in filters: + nb += int(card_obj_iterator(obj_type, filters.get(obj_type, {}))) + return nb + +def card_obj_iterator(obj_type, filters): + if obj_type == 'decoded': + return get_nb_decodeds_objects(filters=filters) + elif obj_type == 'item': + return get_nb_items_objects(filters=filters) + elif obj_type == 'pgp': + return Pgps.nb_all_pgps_objects(filters=filters) def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 5c4b3834..804f6973 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -79,8 +79,11 @@ class Date(object): comp_day = str(computed_date.day).zfill(2) return comp_year + comp_month + comp_day -def get_today_date_str(): - return datetime.date.today().strftime("%Y%m%d") +def get_today_date_str(separator=False): + if separator: + datetime.date.today().strftime("%Y/%m/%d") + else: + return datetime.date.today().strftime("%Y%m%d") def date_add_day(date, num_day=1): new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index 1cd9b784..b6c00c8e 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -19,149 +19,122 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule +from lib.ail_core import get_objects_retro_hunted from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item -from packages import Date +from lib.objects import ail_objects from lib import Tracker -import NotificationHelper # # TODO: refractor - -class Retro_Hunt(AbstractModule): - - # mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}" +class Retro_Hunt_Module(AbstractModule): """ Retro_Hunt module for AIL framework """ def __init__(self): - super(Retro_Hunt, self).__init__() + super(Retro_Hunt_Module, self).__init__() config_loader = ConfigLoader() self.pending_seconds = 5 - self.full_item_url = config_loader.get_config_str("Notifications", "ail_domain") + "/object/item?id=" - # reset on each loop - self.task_uuid = None - self.date_from = 0 - self.date_to = 0 - self.nb_src_done = 0 + self.retro_hunt = None + self.nb_objs = 0 + self.nb_done = 0 self.progress = 0 - self.item = None + self.obj = None self.tags = [] self.redis_logger.info(f"Module: {self.module_name} Launched") - # # TODO: send mails - # # TODO: # start_time # end_time - + # # TODO: # start_time + # # end_time def compute(self, task_uuid): self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}') print(f'starting Retro hunt task {task_uuid}') - self.task_uuid = task_uuid self.progress = 0 # First launch # restart - retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF + self.retro_hunt = Tracker.RetroHunt(task_uuid) - rule = retro_hunt.get_rule(r_compile=True) + rule = self.retro_hunt.get_rule(r_compile=True) + timeout = self.retro_hunt.get_timeout() + self.tags = self.retro_hunt.get_tags() - timeout = retro_hunt.get_timeout() self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') - sources = retro_hunt.get_sources(r_sort=True) - self.date_from = retro_hunt.get_date_from() - self.date_to = retro_hunt.get_date_to() - self.tags = retro_hunt.get_tags() - curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid) - self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources) - self.update_progress(sources, curr_date) - # iterate on date - filter_last = True - while int(curr_date) <= int(self.date_to): - print(curr_date) - dirs_date = Tracker.get_retro_hunt_dir_day_to_analyze(task_uuid, curr_date, filter_last=filter_last, sources=sources) - filter_last = False - nb_id = 0 - self.nb_src_done = 0 - self.update_progress(sources, curr_date) - # # TODO: Filter previous item - for dir in dirs_date: - print(dir) - self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}') - l_obj = Tracker.get_items_to_analyze(dir) - for id in l_obj: - # print(f'{dir} / {id}') - self.item = Item(id) - # save current item in cache - Tracker.set_cache_retro_hunt_task_id(task_uuid, id) + # Filters + filters = self.retro_hunt.get_filters() + if not filters: + filters = {} + for obj_type in get_objects_retro_hunted(): + filters[obj_type] = {} - self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}') + self.nb_objs = ail_objects.card_objs_iterators(filters) - yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, - which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) + # Resume + last_obj = self.retro_hunt.get_last_analyzed() + if last_obj: + last_obj_type, last_obj_subtype, last_obj_id = last_obj.split(':', 2) + else: + last_obj_type = None + last_obj_subtype = None + last_obj_id = None - # save last item - if nb_id % 10 == 0: # # TODO: Add nb before save in DB - Tracker.set_retro_hunt_last_analyzed(task_uuid, id) - nb_id += 1 - self.update_progress(sources, curr_date) + self.nb_done = 0 + self.update_progress() - # PAUSE - self.update_progress(sources, curr_date) - if retro_hunt.to_pause(): - Tracker.set_retro_hunt_last_analyzed(task_uuid, id) - # self.update_progress(sources, curr_date, save_db=True) - retro_hunt.pause() - return None + for obj_type in filters: + if last_obj_type: + filters['start'] = f'{last_obj_subtype}:{last_obj_id}' + last_obj_type = None + for obj in ail_objects.obj_iterator(obj_type, filters): + self.obj = obj + content = obj.get_content(r_str=True) + rule.match(data=content, callback=self.yara_rules_match, + which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) - self.nb_src_done += 1 - self.update_progress(sources, curr_date) - curr_date = Date.date_add_day(curr_date) - print('-----') + self.nb_done += 1 + if self.nb_done % 10 == 0: + self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id()) + self.retro_hunt.set_last_analyzed_cache(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id()) - self.update_progress(sources, curr_date) + # update progress + self.update_progress() - retro_hunt.complete() + # PAUSE + if self.retro_hunt.to_pause(): + self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id()) + self.retro_hunt.pause() + return None + # Completed + self.retro_hunt.complete() print(f'Retro Hunt {task_uuid} completed') self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') - # # TODO: stop - - def update_progress(self, sources, curr_date, save_db=False): - retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF - progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to, - sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done) - if self.progress != progress: - retro_hunt.set_progress(progress) - self.progress = progress - # if save_db: - # Tracker.set_retro_hunt_task_progress(task_uuid, progress) + def update_progress(self): + new_progress = self.nb_done * 100 / self.nb_objs + if int(self.progress) != int(new_progress): + print(new_progress) + self.retro_hunt.set_progress(new_progress) + self.progress = new_progress def yara_rules_match(self, data): - id = self.item.get_id() + obj_id = self.obj.get_id() # print(data) task_uuid = data['namespace'] - self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}') - print(f'Retro hunt {task_uuid} match found: {id}') + self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}') + print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}') - Tracker.save_retro_hunt_match(task_uuid, id) + self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(), obj_id) + # TODO FILTER Tags # Tags for tag in self.tags: msg = f'{tag};{id}' self.add_message_to_queue(msg, 'Tags') # # Mails - # mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) - # if mail_to_notify: - # mail_subject = Tracker.get_email_subject(tracker_uuid) - # mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id) - # for mail in mail_to_notify: - # self.redis_logger.debug(f'Send Mail {mail_subject}') - # print(f'Send Mail {mail_subject}') - # NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + # EXPORTER MAILS return yara.CALLBACK_CONTINUE def run(self): @@ -188,6 +161,5 @@ class Retro_Hunt(AbstractModule): if __name__ == '__main__': - - module = Retro_Hunt() + module = Retro_Hunt_Module() module.run() diff --git a/configs/modules.cfg b/configs/modules.cfg index fc169227..f5be62b2 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -40,7 +40,7 @@ publish = D4_client [D4Client] subscribe = D4_client -[Retro_Hunt] +[Retro_Hunt_Module] publish = Tags [Tracker_Typo_Squatting] diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index e15fae35..77d8df12 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -9,7 +9,7 @@ import os import sys import json -from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -23,6 +23,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib import ail_core +from lib.objects import ail_objects from lib import item_basic from lib import Tracker from lib import Tag @@ -149,6 +150,8 @@ def add_tracked_menu(): galaxies_tags = json.loads(galaxies_tags) except: galaxies_tags = [] + else: + galaxies_tags = [] # custom tags if tags: tags = tags.split() @@ -242,7 +245,7 @@ def tracker_delete(): @login_required @login_read_only def retro_hunt_all_tasks(): - retro_hunts = Tracker.get_retro_hunt_tasks_metas() + retro_hunts = Tracker.get_retro_hunt_metas() return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label) @hunters.route('/retro_hunt/task/show', methods=['GET']) @@ -250,40 +253,35 @@ def retro_hunt_all_tasks(): @login_read_only def retro_hunt_show_task(): task_uuid = request.args.get('uuid', None) + objs = request.args.get('objs', False) - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - if date_from: - date_from = date_from.replace('-', '') - if date_to: - date_to = date_to.replace('-', '') + date_from_item = request.args.get('date_from') + date_to_item = request.args.get('date_to') + if date_from_item: + date_from_item = date_from_item.replace('-', '') + if date_to_item: + date_to_item = date_to_item.replace('-', '') res = Tracker.api_check_retro_hunt_task_uuid(task_uuid) if res: return create_json_response(res[0], res[1]) retro_hunt = Tracker.RetroHunt(task_uuid) - dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'}) + dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'filters', 'nb_objs', 'tags'}) rule_content = Tracker.get_yara_rule_content(dict_task['rule']) + dict_task['filters'] = json.dumps(dict_task['filters'], indent=4) - if date_from: - res = Tracker.api_get_retro_hunt_items({'uuid': task_uuid, 'date_from': date_from, 'date_to': date_to}) - if res[1] != 200: - return create_json_response(res[0], res[1]) - dict_task['items'] = res[0]['items'] - dict_task['date_from_input'] = res[0]['date_from'] - dict_task['date_to_input'] = res[0]['date_to'] + if objs: + dict_task['objs'] = ail_objects.get_objects_meta(retro_hunt.get_objs(), flask_context=True) else: - dict_task['items'] = [] - dict_task['date_from_input'] = dict_task['date_from'] - dict_task['date_to_input'] = dict_task['date_to'] + dict_task['objs'] = [] return render_template("show_retro_hunt.html", dict_task=dict_task, - rule_content=rule_content, - bootstrap_label=bootstrap_label) + rule_content=rule_content, + bootstrap_label=bootstrap_label) -@hunters.route('/retro_hunt/task/add', methods=['GET', 'POST']) +@hunters.route('/retro_hunt/add', methods=['GET', 'POST']) @login_required @login_analyst def retro_hunt_add_task(): @@ -291,23 +289,69 @@ def retro_hunt_add_task(): name = request.form.get("name", '') description = request.form.get("description", '') timeout = request.form.get("timeout", 30) + # TAGS tags = request.form.get("tags", []) + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except: + galaxies_tags = [] + else: + galaxies_tags = [] + # custom tags if tags: tags = tags.split() + escaped_tags = [] + for tag in tags: + escaped_tags.append(escape(tag)) + tags = escaped_tags + else: + tags = [] + tags = tags + taxonomies_tags + galaxies_tags # mails = request.form.get("mails", []) # if mails: # mails = mails.split() - sources = request.form.get("sources", []) - if sources: - sources = json.loads(sources) - - date_from = request.form.get('date_from') - date_to = request.form.get('date_to') - if date_from: - date_from = date_from.replace('-', '') - if date_to: - date_to = date_to.replace('-', '') + # FILTERS + filters = {} + for obj_type in Tracker.get_objects_tracked(): + new_filter = request.form.get(f'{obj_type}_obj') + if new_filter == 'on': + filters[obj_type] = {} + # Date From + date_from = request.form.get(f'date_from_{obj_type}', '').replace('-', '') + if date_from: + filters[obj_type]['date_from'] = date_from + # Date to + date_to = request.form.get(f'date_to_{obj_type}', '').replace('-', '') + if date_to: + filters[obj_type]['date_to'] = date_to + # Mimetypes + mimetypes = request.form.get(f'mimetypes_{obj_type}', []) + if mimetypes: + mimetypes = json.loads(mimetypes) + filters[obj_type]['mimetypes'] = mimetypes + # Sources + sources = request.form.get(f'sources_{obj_type}', []) + if sources: + sources = json.loads(sources) + filters[obj_type]['sources'] = sources + # Subtypes + for obj_subtype in ail_core.get_object_all_subtypes(obj_type): + subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}') + if subtype == 'on': + if 'subtypes' not in filters[obj_type]: + filters[obj_type]['subtypes'] = [] + filters[obj_type]['subtypes'].append(obj_subtype) # YARA # yara_default_rule = request.form.get("yara_default_rule") @@ -322,9 +366,9 @@ def retro_hunt_add_task(): user_id = current_user.get_id() input_dict = {"name": name, "description": description, "creator": user_id, - "rule": rule, "type": rule_type, - "tags": tags, "sources": sources, "timeout": timeout, #"mails": mails, - "date_from": date_from, "date_to": date_to} + "rule": rule, "type": rule_type, + "tags": tags, "filters": filters, "timeout": timeout, # "mails": mails + } res = Tracker.api_create_retro_hunt_task(input_dict, user_id) if res[1] == 200: @@ -334,8 +378,9 @@ def retro_hunt_add_task(): return create_json_response(res[0], res[1]) else: return render_template("add_retro_hunt_task.html", - all_yara_files=Tracker.get_all_default_yara_files(), - all_sources=item_basic.get_all_items_sources(r_list=True)) + all_yara_files=Tracker.get_all_default_yara_files(), + tags_selector_data=Tag.get_tags_selector_data(), + items_sources=item_basic.get_all_items_sources(r_list=True)) @hunters.route('/retro_hunt/task/pause', methods=['GET']) @login_required @@ -368,27 +413,4 @@ def retro_hunt_delete_task(): return redirect(url_for('hunters.retro_hunt_all_tasks')) -#### JSON #### - -@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET']) -@login_required -@login_read_only -def get_json_retro_hunt_nb_items_by_date(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - - if date_from: - date_from = date_from.replace('-', '') - if date_to: - date_to = date_to.replace('-', '') - - task_uuid = request.args.get('uuid') - - if date_from and date_to: - res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to) - else: - res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid]) - return jsonify(res) - - ## - - ## diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index e999f280..c71a91c9 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -127,6 +127,7 @@ def show_tracker(): meta['date_to'] = date_to print(meta['filters']) meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', [])) + # meta['filters'] = json.dumps(meta['filters'], indent=4) return render_template("showTracker.html", tracker_metadata=meta, yara_rule_content=yara_rule_content, diff --git a/var/www/templates/hunter/add_retro_hunt_task.html b/var/www/templates/hunter/add_retro_hunt_task.html index b999af91..4dc87e27 100644 --- a/var/www/templates/hunter/add_retro_hunt_task.html +++ b/var/www/templates/hunter/add_retro_hunt_task.html @@ -31,28 +31,22 @@
-
-
-
Create a new Retro Hunt task
-
-
+
+
+
Create a new Retro Hunt task
+
+
-
+ -
-
-
-
-
-
- -
-
-
-
-
- -
+
+
+
+
+
+
+ +
-
-
-
-
- -
+
+
+
+
+ +
-
-
-
-
- -
+
+
+ Objects to Track: +
+
+{#
#} +{# #} +{# #} +{#
#} +{#
#} +{# #} +{# #} +{#
#} +
+ + +
+{#
#} +{# #} +{# #} +{#
#} +
+ + +
+
+
+
Filter Item by sources
+
+
+
+
+ +
+
Date range:
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
-
Date range:
-
-
-
-
- -
-
-
-
-
- -
-
-
-
-
+{#
#} +{# #} +{# #} +{#
#} +{#
#} +{#
#} +{#
Filter PGP by subtype:
#} +{#
#} +{# #} +{# #} +{#
#} +{#
#} +{# #} +{# #} +{#
#} +{#
#} +{#
#} -
-
+ +
+
+ +
+
+ Tags +
+
+
+
+
+
+ +
+ {% include 'tags/block_tags_selector.html' %} +
+
+ + +
+
+ +
+
-
-
+
+
-
Default YARA rules:
- +
Default YARA rules:
+ - 

+                                    

 
-										

+                                

 
-										

+                                

 
-										
Custom YARA rules:

-										

-											
-										

-									

+                                
Custom YARA rules:

+                                

+                                    
+                                

+                            

 
-									

-									
+                            

+                            
 
-								

+                        
 
 
 
-						  

-						

+                    

+                

 
 
 			

-
 		
 	
 
@@ -177,22 +254,21 @@ $(document).ready(function(){
 			}
 	});
 
-	sources = $('#sources').tagSuggest({
-		data: {{all_sources|safe}},
-		{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
+	sources_item = $('#sources_item').tagSuggest({
+		data: {{items_sources|safe}},
 		sortOrder: 'name',
 		maxDropHeight: 200,
-		name: 'sources',
-		emptyText: 'Sources to track (ALL IF EMPTY)',
+		name: 'sources_item',
+		emptyText: 'Item Sources to track (ALL IF EMPTY)',
 		});
 
-	{%if dict_tracker%}
-		$('#tracker_type').val('{{dict_tracker['type']}}').change();
+    $('#item_obj').on("change", function () {
+		item_source_input_controller();
+	});
+    /*$('#pgp_obj').on("change", function () {
+		pgp_source_input_controller();
+	});*/
 
-		{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
-			$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
-		{%endif%}
-	{%endif%}
 
 });
 
@@ -210,7 +286,21 @@ function toggle_sidebar(){
 	}
 }
 
+function item_source_input_controller() {
+	if($('#item_obj').is(':checked')){
+		$("#sources_item_div").show();
+	}else{
+		$("#sources_item_div").hide();
+	}
+}
 
+function pgp_source_input_controller() {
+	if($('#pgp_obj').is(':checked')){
+		$("#sources_pgp_div").show();
+	}else{
+		$("#sources_pgp_div").hide();
+	}
+}
 
 function get_default_rule_content(selector){
 	var yara_name = selector.value
diff --git a/var/www/templates/hunter/show_retro_hunt.html b/var/www/templates/hunter/show_retro_hunt.html
index 855c187f..ebb65f0e 100644
--- a/var/www/templates/hunter/show_retro_hunt.html
+++ b/var/www/templates/hunter/show_retro_hunt.html
@@ -20,7 +20,6 @@
 		
 		
 		
-		
 		
 		
 
@@ -48,9 +47,9 @@
 
 				

 
-					

-					  

-					    

+                    

+                        

+                            

 								

 									

 										
@@ -68,13 +67,13 @@
 										

 											{{dict_task['name']}}
 											{%if dict_task['state']!='completed'%}
-												[{{ dict_task['progress']}}%]
+												[{{ dict_task['progress']}}%]
 											{%endif%}
 										

 
 									

 								

-					      

+					            

 									

 										{%if dict_task['state']=='paused'%}
 											
@@ -94,83 +93,85 @@
 										
 									

 									

-					        	
-											
-												
-													
-													
-												
-												
-													
-													
-												
-												
-													
-													
-												
-												
-													
-													
-												
-												
-													
-													
-												
-												
-													
-													
-												
-											
-										
Date
-														{{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
-													
Search date
-														{{dict_task['date_from'][0:4]}}/{{dict_task['date_from'][4:6]}}/{{dict_task['date_from'][6:8]}} -
-														{{dict_task['date_to'][0:4]}}/{{dict_task['date_to'][4:6]}}/{{dict_task['date_to'][6:8]}}
-													
Description{{dict_task['description']}}
Tags
-														{%for tag in dict_task['tags']%}
-															
-																{{ tag }}
-															
-														{%endfor%}
-													
Creator{{dict_task['creator']}}
Sources
-														

-															{%if not dict_task['sources']%}
-																All Souces
-															{%else%}
-																{%for source in dict_task['sources']%}
-																	{{source}}
-																{%endfor%}
-															{%endif%}
-														

-													

-					      

-					    

-					  

+					        	    
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                            
+                                            
+                                        
+                                        
+                                    
Date
+                                                {{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
+                                            
Description{{dict_task['description']}}
Tags
+                                                {%for tag in dict_task['tags']%}
+                                                    
+                                                        {{ tag }}
+                                                    
+                                                {%endfor%}
+                                            
Creator{{dict_task['creator']}}
Filters
+                                                

+                                                    {% if dict_task['filters'] %}
+                                                        
{{ dict_task['filters'] }}

+                                                    {% else %}
+                                                        No Filters

+                                                    {% endif %}
+                                                

+                                            
Objects Match
+                                                {%for obj_type in dict_task['nb_objs']%}
+                                                    

+                                                        {{ obj_type }}
+                                                        {{ dict_task['nb_objs'][obj_type] }}
+                                                    

+                                                {%endfor%}
+                                            

+					            

+					        

+                        

 
-					  

+					    

 
 							

 								

 
-									

-										

-											

-												

-												
-											

-										

-										

-											

-												

-												
-											

-										

-									

+{#									
#}
+{#										
#}
+{#											
#}
+{#												
#}
+{#												 #}
+{#													value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}"#}
+{#											
#}
+{#										
#}
+{#										
#}
+{#											
#}
+{#												
#}
+{#												 #}
+{#													value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}"#}
+{#											
#}
+{#										
#}
+{#									
#}
 
 									
 
 								

@@ -178,40 +179,52 @@
 
 							
{{ rule_content }}

 
-					  

+					    

 					

 
-					{%if dict_task['items']%}
+					{% if dict_task['objs'] %}
 						

-							
+							

-										
-										
+										
+                                        
+                                        
+                                        
+                                        
-
-									{% for item in dict_task['items'] %}
-										
-											
-											
-										
+									{% for object in dict_task['objs'] %}
+                                        
+                                            
+                                            
+                                            
+                                            
+                                            
+                                        
 									{%  endfor %}
-
 								

 								
 									
DateItem IdTypeIdTags
 									

 								
 								

-												{{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}}
-											
-												
-													
{{ item['id'] }}

-												
-												

-					                {% for tag in item['tags'] %}
-						                
-						                  {{ tag }}
-						                
-					                {%  endfor %}
-					              

-											

+                                                {% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
+                                                    {% include 'objects/obj_svg_block.html' %}
+                                                {% endwith %}
+                                                {{ object['type']}}
+                                            
+                                                {% if object['subtype'] %}
+                                                    {{ object['subtype']}}
+                                                {% endif %}
+                                            
+                                                
+                                                    {{ object['id']}}
+                                                
+                                            
+                                                {% for tag in object['tags'] %}
+                                                    {{ tag }}
+                                                {% endfor %}
+                                            
+            {#									#}
+            {#										#}
+            {#									#}
+                                            

 							

 						

@@ -219,13 +232,6 @@
 						

 					{% endif %}
 
-
-					

-					

-					

-					

-
-
 				

 			

 		

@@ -234,7 +240,7 @@
 $(document).ready(function(){
 		$('#nav_title_retro_hunt').removeClass("text-muted");
 
-		$('#date-range-from').dateRangePicker({
+		/*$('#date-range-from').dateRangePicker({
 			separator : ' to ',
 			getValue: function(){
 					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
@@ -259,18 +265,15 @@ $(document).ready(function(){
 						$('#date-range-from-input').val(s1);
 						$('#date-range-to-input').val(s2);
 				}
-		});
+		});*/
 
-		$('#items_table').DataTable({
+        {% if dict_task['objs'] %}
+		$('#objs_table').DataTable({
 	        "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
 	        "iDisplayLength": 10,
 	        "order": [[ 0, "asc" ]]
-	  });
-
-		let div_width = $("#graphline").width();
-		$.getJSON( "{{ url_for('hunters.get_json_retro_hunt_nb_items_by_date') }}?uuid={{ dict_task['uuid'] }}&date_from={{ dict_task['date_from_input'] }}&date_to={{ dict_task['date_to_input'] }}",
-			function( data ) {multilines_group("graphline", data, {"width": div_width});}
-		);
+	    });
+        {% endif%}
 
 });
 
@@ -291,7 +294,8 @@ function toggle_sidebar(){
 function getItems() {
 	var date_from = $('#date-range-from-input').val();
 	var date_to =$('#date-range-to-input').val();
-	window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
+	{#window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);#}
+    window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&objs=True");
 }