diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index 3de267ed..bd6f2ffc 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -233,17 +233,61 @@ def ail_2_ail_migration(): # item in queue ail_2_ail.set_last_updated_sync_config() +############################### +# # +# TRACKER MIGRATION # +# # +############################### + +def get_tracker_level(tracker_uuid): + level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level') + if not level: + level = 0 + return int(level) + +def get_tracker_metadata(tracker_uuid): + meta = {'uuid': tracker_uuid, + 'tracked': r_serv_tracker.hget('tracker:{tracker_uuid}', 'tracked'), + 'type': r_serv_tracker.hget('tracker:{tracker_uuid}', 'type'), + 'date': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date'), + 'first_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen'), + 'last_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen'), + 'user_id': r_serv_tracker.hget('tracker:{tracker_uuid}', 'user_id'), + 'level': get_tracker_level(tracker_uuid), + 'mails': list(r_serv_tracker.smembers('tracker:mail:{tracker_uuid}')), + 'sources': list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')), + 'tags': list(r_serv_tracker.smembers(f'tracker:tags:{tracker_uuid}')), + 'description': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'description'), + 'webhook': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')} + return meta + +def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to): + all_item_id = set() + if date_from and date_to: + l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True) + if l_date_match: + dict_date_match = dict(l_date_match) + for date_day in Date.substract_date(date_from, date_to): + if date_day in dict_date_match: + all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') + return all_item_id + # trackers + retro_hunts def trackers_migration(): print('TRACKERS MIGRATION...') for tracker_uuid in old_Tracker.get_all_tracker_uuid(): - meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True) - Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen']) + meta = get_tracker_metadata(tracker_uuid) + Tracker._re_create_tracker(meta['type'], meta['uuid'], meta['tracked'], meta['user_id'], meta['level'], + tags=meta['tags'], mails=meta['mails'], description=meta['description'], + webhook=meta['webhook'], sources=meta['sources'], + first_seen=meta['first_seen'], last_seen=meta['last_seen']) + tracker = Tracker.Tracker(tracker_uuid) # object migration # # TODO: in background for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']): print(item_id) - Tracker.add_tracked_item(tracker_uuid, item_id) + item_date = get_item_date(item_id) + tracker.add('item', '', item_id, date=item_date) print('RETRO HUNT MIGRATION...') @@ -929,13 +973,13 @@ if __name__ == '__main__': # user_migration() #tags_migration() # items_migration() - crawler_migration() + # crawler_migration() # domain_migration() # TO TEST ########################### # decodeds_migration() # screenshots_migration() # subtypes_obj_migration() # ail_2_ail_migration() - # trackers_migration() + trackers_migration() # investigations_migration() ## statistics_migration() diff --git a/bin/core/DbCleaner.py b/bin/core/DbCleaner.py index 4be8579f..721e38cc 100755 --- a/bin/core/DbCleaner.py +++ b/bin/core/DbCleaner.py @@ -14,23 +14,9 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from packages import Date -from packages import Term from pubsublogger import publisher -def clean_term_db_stat_token(): - all_stat_date = Term.get_all_token_stat_history() - - list_date_to_keep = Date.get_date_range(31) - for date in all_stat_date: - if date not in list_date_to_keep: - # remove history - Term.delete_token_statistics_by_date(date) - - print('Term Stats Cleaned') - - if __name__ == "__main__": publisher.port = 6380 @@ -46,7 +32,7 @@ if __name__ == "__main__": while True: if daily_cleaner: - clean_term_db_stat_token() + daily_cleaner = False else: sys.exit(0) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index c48e86cb..4a5ab6c2 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -105,7 +105,7 @@ class MailExporterTracker(MailExporter): def export(self, tracker, obj): # TODO match tracker_type = tracker.get_type() - tracker_name = tracker.get_tracker() + tracker_name = tracker.get_tracked() subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" body += f'Item: {obj.id}\nurl:{obj.get_link()}' @@ -115,4 +115,4 @@ class MailExporterTracker(MailExporter): # body += f'Tracker Match:\n\n{escape(match)}' for mail in tracker.get_mails(): - self._export(mail, subject, body) \ No newline at end of file + self._export(mail, subject, body) diff --git a/bin/exporter/WebHookExporter.py b/bin/exporter/WebHookExporter.py index 1322a8fa..b4391922 100755 --- a/bin/exporter/WebHookExporter.py +++ b/bin/exporter/WebHookExporter.py @@ -53,7 +53,7 @@ class WebHookExporterTracker(WebHookExporter): data = {'trackerId': tracker.get_uuid(), 'trackerType': tracker.get_type(), 'tags': tracker.get_tags(), - 'tracker': tracker.get_tracker(), + 'tracker': tracker.get_tracked(), # object 'itemId': obj.get_id(), 'itemURL': obj.get_link()} diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index bab8cd89..09c84f98 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -186,9 +186,9 @@ class Investigation(object): def set_threat_level(self, threat_level): try: threat_level = int(threat_level) - except: + except TypeError: raise UpdateInvestigationError('threat_level Not an integer') - if threat_level >= 1 and threat_level <= 4: + if 1 <= threat_level <= 4: r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level) else: raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}') @@ -196,9 +196,9 @@ class Investigation(object): def set_analysis(self, analysis): try: analysis = int(analysis) - except: + except TypeError: raise UpdateInvestigationError('analysis Not an integer') - if analysis >= 0 and analysis <= 2: + if 0 <= analysis <= 2: r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis) else: raise UpdateInvestigationError(f'Invalid analysis: {analysis}') diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index e0381f97..b474116b 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* - +import json import os import re import sys @@ -13,14 +13,17 @@ import base64 from ail_typo_squatting import runAll import math - +from collections import defaultdict from flask import escape +from textblob import TextBlob +from nltk.tokenize import RegexpTokenizer sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from packages import Date +from lib.ail_core import get_objects_tracked, get_object_all_subtypes from lib import ConfigLoader from lib import item_basic from lib import Tag @@ -44,6 +47,10 @@ email_regex = re.compile(email_regex) special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\') special_characters.add('\\s') +# NLTK tokenizer +tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+', + gaps=True, discard_empty=True) + ############### #### UTILS #### def is_valid_uuid_v4(UUID): @@ -93,15 +100,74 @@ class Tracker: def exists(self): return r_tracker.exists(f'tracker:{self.uuid}') + def _set_field(self, field, value): + r_tracker.hset(f'tracker:{self.uuid}', field, value) + def get_date(self): return r_tracker.hget(f'tracker:{self.uuid}', 'date') + def get_last_change(self, r_str=False): + last_change = r_tracker.hget(f'tracker:{self.uuid}', 'last_change') + if r_str and last_change: + last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S') + return last_change + def get_first_seen(self): return r_tracker.hget(f'tracker:{self.uuid}', 'first_seen') def get_last_seen(self): return r_tracker.hget(f'tracker:{self.uuid}', 'last_seen') + def _set_first_seen(self, date): + self._set_field('first_seen', date) + + def _set_last_seen(self, date): + self._set_field('last_seen', date) + + def _exist_date(self, date): + return r_serv_tracker.exists(f'tracker:objs:{self.uuid}:{date}') + + # TODO: ADD CACHE ??? + def update_daterange(self, date=None): + first_seen = self.get_first_seen() + # Added Object + if date: + date = int(date) + first_seen = self.get_first_seen() + # if op == 'add': + if not first_seen: + self._set_first_seen(date) + self._set_last_seen(date) + else: + first_seen = int(first_seen) + last_seen = int(self.get_last_seen()) + if date < first_seen: + self._set_first_seen(date) + if date > last_seen: + self._set_last_seen(date) + else: + last_seen = self.get_last_seen() + if first_seen and last_seen: + valid_first_seen = self._exist_date(first_seen) + valid_last_seen = self._exist_date(last_seen) + # update first seen + if not valid_first_seen: + for date in Date.get_daterange(first_seen, last_seen): + if self._exist_date(date): + self._set_first_seen(date) + valid_first_seen = True + break + # update last seen + if not valid_last_seen: + for date in reversed(Date.get_daterange(first_seen, last_seen)): + if self._exist_date(date): + self._set_first_seen(date) + valid_last_seen = True + break + if not valid_first_seen or not valid_last_seen: + r_tracker.hdel(f'tracker:{self.uuid}', 'first_seen') + r_tracker.hdel(f'tracker:{self.uuid}', 'last_seen') + def get_description(self): return r_tracker.hget(f'tracker:{self.uuid}', 'description') @@ -111,10 +177,37 @@ class Tracker: level = 0 return int(level) - def get_sources(self): - return r_tracker.smembers(f'tracker:sources:{self.uuid}') + def is_level_user(self): + return self.get_level() == 0 - def get_tracker(self): + def is_level_global(self): + return self.get_level() == 1 + + def _set_level(self, level, tracker_type=None, user=None): + if not tracker_type: + tracker_type = self.get_type() + if level == 0: # user only + if not user: + user = self.get_user() + r_serv_tracker.sadd(f'user:tracker:{user}', self.uuid) + r_serv_tracker.sadd(f'user:tracker:{user}:{tracker_type}', self.uuid) + elif level == 1: # global + r_serv_tracker.sadd('global:tracker', self.uuid) + r_serv_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid) + self._set_field('level', level) + + def get_filters(self): + filters = r_tracker.hget(f'tracker:{self.uuid}', 'filters') + if not filters: + return {} + else: + return json.loads(filters) + + def set_filters(self, filters): + if filters: + self._set_field('filters', json.dumps(filters)) + + def get_tracked(self): return r_tracker.hget(f'tracker:{self.uuid}', 'tracked') def get_type(self): @@ -123,12 +216,22 @@ class Tracker: def get_tags(self): return r_tracker.smembers(f'tracker:tags:{self.uuid}') + def _set_tags(self, tags): + for tag in tags: + tag = escape(tag) + r_serv_tracker.sadd(f'tracker:tags:{self.uuid}', tag) + Tag.create_custom_tag(tag) # TODO CUSTOM TAGS + def mail_export(self): return r_tracker.exists(f'tracker:mail:{self.uuid}') def get_mails(self): return r_tracker.smembers(f'tracker:mail:{self.uuid}') + def _set_mails(self, mails): + for mail in mails: + r_serv_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail)) + def get_user(self): return r_tracker.hget(f'tracker:{self.uuid}', 'user_id') @@ -138,6 +241,20 @@ class Tracker: def get_webhook(self): return r_tracker.hget(f'tracker:{self.uuid}', 'webhook') + def get_sparkline(self, nb_day=6): + date_range_sparkline = Date.get_date_range(nb_day) + sparkline = [] + for date in date_range_sparkline: + nb_seen_this_day = self.get_nb_objs_by_date(date) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + sparkline.append(int(nb_seen_this_day)) + return sparkline + + def get_rule(self): + yar_path = self.get_tracked() + return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path)) + # TODO get objects/ tracked items @@ -146,7 +263,7 @@ class Tracker: if not options: options = set() meta = {'uuid': self.uuid, - 'tracker': self.get_tracker(), + 'tracked': self.get_tracked(), # TODO TO CHECK 'type': self.get_type(), 'date': self.get_date(), 'first_seen': self.get_first_seen(), @@ -159,308 +276,443 @@ class Tracker: meta['description'] = self.get_description() if 'tags' in options: meta['tags'] = self.get_tags() - if 'sources' in options: - meta['sources'] = self.get_sources() + if 'filters' in options: + meta['filters'] = self.get_filters() if 'mails' in options: meta['mails'] = self.get_mails() if 'webhooks' in options: meta['webhook'] = self.get_webhook() - # if 'sparkline' in options: - # meta['sparkline'] = get_tracker_sparkline(tracker_uuid) + if 'sparkline' in options: + meta['sparkline'] = self.get_sparkline(6) + return meta + def _add_to_dashboard(self, obj_type, subtype, obj_id): + mess = f'{self.uuid}:{int(time.time())}:{obj_type}:{subtype}:{obj_id}' + if self.is_level_user(): + user = self.get_user() + r_serv_tracker.lpush(f'trackers:user:{user}', mess) + r_serv_tracker.ltrim(f'trackers:user:{user}', 0, 9) + else: + r_serv_tracker.lpush('trackers:dashboard', mess) + r_serv_tracker.ltrim(f'trackers:dashboard', 0, 9) + # - TODO Data Retention TO Implement - # + # Or Daily/Monthly Global DB Cleanup: + # Iterate on each tracker: + # Iterate on each Obj: + # Iterate on each date: + # Delete from tracker range if date limit exceeded + # - TODO + def add(self, obj_type, subtype, obj_id, date=None): + if not subtype: + subtype = '' + if not date: + date = Date.get_today_date_str() + new_obj_date = r_serv_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') + new_obj = r_serv_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) + # MATCHES + if new_obj: + r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', 1, 'total') + r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', 1, obj_type) - # TODO - def add(self, obj_id): + # Only save date for daterange objects - Needed for the DB Cleaner + if obj_type != 'item': # not obj_date: + r_serv_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) + r_serv_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') + + if new_obj_date: + self.update_daterange(date) + + self._add_to_dashboard(obj_type, subtype, obj_id) + + def get_objs_by_type(self, obj_type): + return r_serv_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}') + + def get_nb_objs_by_date(self, date): + return r_serv_tracker.scard(f'tracker:objs:{self.uuid}:{date}') + + def get_objs_by_date(self, date): + return r_serv_tracker.smembers(f'tracker:objs:{self.uuid}:{date}') + + def get_objs_by_daterange(self, date_from, date_to): + objs = set() + for date in Date.get_daterange(date_from, date_to): + objs |= self.get_objs_by_date(date) + return objs + + def get_obj_dates(self, obj_type, subtype, obj_id): + if obj_type == 'item': + return [item_basic.get_item_date(obj_id)] + else: + return r_serv_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}') + + def remove(self, obj_type, subtype, obj_id): + if not subtype: + subtype = '' + + for date in self.get_obj_dates(obj_type, subtype, obj_id): + r_serv_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') + r_serv_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) + + r_serv_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) + r_serv_tracker.srem(f'tracker:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') + # MATCHES + r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', -1, 'total') + r_serv_tracker.zincrby(f'tracker:match:{self.uuid}', -1, obj_type) + self.update_daterange() + + # TODO escape tags ???? + # TODO escape mails ???? + def create(self, tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None): + if self.exists(): + raise Exception('Error: Tracker already exists') + + # YARA + if tracker_type == 'yara_custom' or tracker_type == 'yara_default': + to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid) + tracker_type = 'yara' + + elif tracker_type == 'typosquatting': + domain = to_track.split(" ")[0] + typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1 + for typo in typo_generation: + r_serv_tracker.sadd(f'tracker:typosquatting:{to_track}', typo) + + # create metadata + self._set_field('tracked', to_track) + self._set_field('type', tracker_type) + self._set_field('date', datetime.date.today().strftime("%Y%m%d")) + self._set_field('user_id', user_id) + if description: + self._set_field('description', escape(description)) + if webhook: + self._set_field('webhook', webhook) + + # create all tracker set + r_serv_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # TODO RENAME ???? + # create tracker - uuid map + r_serv_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) # TODO RENAME ???? + r_serv_tracker.sadd('trackers:all', self.uuid) + r_serv_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) + + # TRACKER LEVEL + self._set_level(level, tracker_type=tracker_type, user=user_id) + + # create tracker tags list + if tags: + self._set_tags(tags) + + # create tracker mail notification list + if mails: + self._set_mails(mails) + + # TODO Delete filters + # Filters + if not filters: + filters = {} + for obj_type in get_objects_tracked(): + filters[obj_type] = {} + else: + self.set_filters(filters) + for obj_type in filters: + r_serv_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) + r_serv_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') + + self._set_field('last_change', time.time()) + + # toggle refresh module tracker list/set + trigger_trackers_refresh(tracker_type) + return self.uuid + + def edit(self, tracker_type, to_track, level, description=None, filters={}, tags=[], mails=[], webhook=None): + + # edit tracker + old_type = self.get_type() + old_to_track = self.get_tracked() + old_level = self.get_level() + user_id = self.get_user() + + # YARA + if tracker_type == 'yara_custom' or tracker_type == 'yara_default': + # create yara rule + if tracker_type == 'yara_default' and old_type == 'yara': + if not is_default_yara_rule(old_to_track): + filepath = get_yara_rule_file_by_tracker_name(old_to_track) + if filepath: + os.remove(filepath) + to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid) + tracker_type = 'yara' + + # TODO TYPO EDIT + elif tracker_type == 'typosquatting': + pass + + if tracker_type != old_type: + # LEVEL + if old_level == 0: + r_serv_tracker.srem(f'user:tracker:{user_id}:{old_type}', self.uuid) + elif old_level == 1: + r_serv_tracker.srem(f'global:tracker:{old_type}', self.uuid) + self._set_level(level, tracker_type=tracker_type, user=user_id) + # Delete OLD YARA Rule File + if old_type == 'yara': + if not is_default_yara_rule(old_to_track): + filepath = get_yara_rule_file_by_tracker_name(old_to_track) + if filepath: + os.remove(filepath) + self._set_field('type', tracker_type) + + # create all tracker set + r_serv_tracker.srem(f'all:tracker:{old_type}', old_to_track) + r_serv_tracker.sadd(f'all:tracker:{tracker_type}', to_track) + # create tracker - uuid map + r_serv_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid) + r_serv_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) + # create all tracker set by type + r_serv_tracker.srem(f'trackers:all:{old_type}', self.uuid) + r_serv_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) + + # Same Type + elif level != old_level: + if level == 0: + r_serv_tracker.srem('global:tracker', self.uuid) + elif level == 1: + r_serv_tracker.srem(f'user:tracker:{user_id}', self.uuid) + self._set_level(level, tracker_type=tracker_type, user=user_id) + + # To Track Edited + if to_track != old_to_track: + self._set_field('tracked', to_track) + + self._set_field('description', description) + self._set_field('webhook', webhook) + + # Tags + nb_old_tags = r_serv_tracker.scard(f'tracker:tags:{self.uuid}') + if nb_old_tags > 0 or tags: + r_serv_tracker.delete(f'tracker:tags:{self.uuid}') + self._set_tags(tags) + + # Mails + nb_old_mails = r_serv_tracker.scard(f'tracker:mail:{self.uuid}') + if nb_old_mails > 0 or mails: + r_serv_tracker.delete(f'tracker:mail:{self.uuid}') + self._set_mails(mails) + + nb_old_sources = r_serv_tracker.scard(f'tracker:sources:{self.uuid}') # TODO FILTERS + if nb_old_sources > 0 or sources: + r_serv_tracker.delete(f'tracker:sources:{self.uuid}') + self._set_sources(sources) + + # Refresh Trackers + trigger_trackers_refresh(tracker_type) + if tracker_type != old_type: + trigger_trackers_refresh(old_type) + + self._set_field('last_change', time.time()) + return self.uuid + + def delete(self): pass -################################################################################################ -################################################################################################ -################################################################################################ -def get_all_tracker_type(): - return ['word', 'set', 'regex', 'yara'] +def create_tracker(tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, tracker_uuid=None): + if not tracker_uuid: + tracker_uuid = str(uuid.uuid4()) + tracker = Tracker(tracker_uuid) + return tracker.create(tracker_type, to_track, user_id, level, description=description, filters=filters, tags=tags, + mails=mails, webhook=webhook) -def get_all_tracker_uuid(): +def _re_create_tracker(tracker_type, tracker_uuid, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, first_seen=None, last_seen=None): + create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters, + tags=tags, mails=mails, webhook=webhook, tracker_uuid=tracker_uuid) + +def get_trackers_types(): + return ['word', 'set', 'regex', 'typosquatting', 'yara'] + +def get_trackers(): return r_serv_tracker.smembers(f'trackers:all') -def get_all_tracker_uuid_by_type(tracker_type): +def get_trackers_by_type(tracker_type): return r_serv_tracker.smembers(f'trackers:all:{tracker_type}') -# def get_all_tracker(): -# l_keys_name = [] -# for tracker_type in get_all_tracker_type(): -# l_keys_name.append(f'all:tracker:{tracker_type}') -# return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:]) +def _get_tracked_by_obj_type(tracker_type, obj_type): + return r_serv_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}') -def get_all_tracker_by_type(tracker_type): +def get_trackers_by_tracked_obj_type(tracker_type, obj_type, tracked): + trackers_uuid = set() + for res in r_serv_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'): + tracker_uuid, tracker_obj_type = res.split(':', 1) + if tracker_obj_type == obj_type: + trackers_uuid.add(tracker_uuid) + return trackers_uuid + +def get_trackers_by_tracked(tracker_type, tracked): + return r_serv_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}') + +def get_user_trackers_by_tracked(tracker_type, tracked, user_id): + user_trackers = get_user_trackers(user_id, tracker_type=tracker_type) + trackers_uuid = get_trackers_by_tracked(tracker_type, tracked) + return trackers_uuid.intersection(user_trackers) + +def get_trackers_tracked_by_type(tracker_type): return r_serv_tracker.smembers(f'all:tracker:{tracker_type}') -def get_user_trackers_uuid(user_id, tracker_type=None): - if tracker_type: - return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') - else: - return r_serv_tracker.smembers(f'user:tracker:{user_id}') - -def get_global_trackers_uuid(tracker_type=None): +def get_global_trackers(tracker_type=None): if tracker_type: return r_serv_tracker.smembers(f'global:tracker:{tracker_type}') else: return r_serv_tracker.smembers('global:tracker') -def get_tracker_by_uuid(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked') - -def get_tracker_type(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'type') - -def get_tracker_level(tracker_uuid): - level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level') - if not level: - level = 0 - return int(level) - -def get_tracker_user_id(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') - -def get_tracker_uuid_list(tracker, tracker_type): ######################################################### USE ME - return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker))) - -def get_tracker_tags(tracker_uuid): - return list(r_serv_tracker.smembers('tracker:tags:{}'.format(tracker_uuid))) - -def get_tracker_mails(tracker_uuid): - return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid))) - -def get_tracker_webhook(tracker_uuid): - return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook') - -def get_tracker_uuid_sources(tracker_uuid): - return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')) - -def get_tracker_description(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description') - -def get_tracker_date(tracker_uuid): - return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date') - -def get_tracker_first_seen(tracker_uuid): - res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen') - if res: - return res +def get_user_trackers(user_id, tracker_type=None): + if tracker_type: + return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') else: - return None + return r_serv_tracker.smembers(f'user:tracker:{user_id}') -def get_tracker_last_seen(tracker_uuid): - res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen') - if res: - return res +def get_nb_global_trackers(tracker_type=None): + if tracker_type: + return r_serv_tracker.scard(f'global:tracker:{tracker_type}') else: - return None + return r_serv_tracker.scard('global:tracker') -def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sources=True, sparkline=False, webhook=False): - dict_uuid = {} - dict_uuid['uuid'] = tracker_uuid - dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid) - dict_uuid['type'] = get_tracker_type(tracker_uuid) - dict_uuid['date'] = get_tracker_date(tracker_uuid) - dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid) - dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid) - if user_id: - dict_uuid['user_id'] = get_tracker_user_id(tracker_uuid) - if level: - dict_uuid['level'] = get_tracker_level(tracker_uuid) - if mails: - dict_uuid['mails'] = get_tracker_mails(tracker_uuid) - if sources: - dict_uuid['sources'] = get_tracker_uuid_sources(tracker_uuid) - if tags: - dict_uuid['tags'] = get_tracker_tags(tracker_uuid) - if sparkline: - dict_uuid['sparkline'] = get_tracker_sparkline(tracker_uuid) - if description: - dict_uuid['description'] = get_tracker_description(tracker_uuid) - if webhook: - dict_uuid['webhook'] = get_tracker_webhook(tracker_uuid) +def get_nb_user_trackers(user_id, tracker_type=None): + if tracker_type: + return r_serv_tracker.scard(f'user:tracker:{user_id}:{tracker_type}') + else: + return r_serv_tracker.scard(f'user:tracker:{user_id}') - return dict_uuid +def get_user_trackers_meta(user_id, tracker_type=None): + metas = [] + for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type): + tracker = Tracker(tracker_uuid) + metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + return metas -def get_user_trackers_metadata(user_id, tracker_type=None): - meta_trackers = [] - for tracker_uuid in get_user_trackers_uuid(user_id, tracker_type=None): - meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True)) - return meta_trackers +def get_global_trackers_meta(tracker_type=None): + metas = [] + for tracker_uuid in get_global_trackers(tracker_type=tracker_type): + tracker = Tracker(tracker_uuid) + metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + return metas -def get_global_trackers_metadata(tracker_type=None): - meta_trackers = [] - for tracker_uuid in get_global_trackers_uuid(tracker_type=None): - meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True)) - return meta_trackers - -def get_tracker_metadata_api(request_dict): - tracker_uuid = request_dict.get('tracker_uuid', None) - if not request_dict: - return {'status': 'error', 'reason': 'Malformed JSON'}, 400 - if not tracker_uuid: - return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 - if not is_valid_uuid_v4(tracker_uuid): - return {"status": "error", "reason": "Invalid Tracker UUID"}, 400 - if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): - return {'status': 'error', 'reason': 'Tracker not found'}, 404 - - dict_tracker = {'status': 'success', - 'uuid': tracker_uuid, - 'user_id': get_tracker_user_id(tracker_uuid), - 'tracker': get_tracker_by_uuid(tracker_uuid), - 'type': get_tracker_type(tracker_uuid), - 'date': get_tracker_date(tracker_uuid), - 'first_seen': get_tracker_first_seen(tracker_uuid), - 'last_seen': get_tracker_last_seen(tracker_uuid), - 'level': get_tracker_level(tracker_uuid), - 'mails': get_tracker_mails(tracker_uuid), - 'tags': get_tracker_tags(tracker_uuid), - 'description': get_tracker_description(tracker_uuid), - 'webhook': get_tracker_webhook(tracker_uuid) - } - - return dict_tracker, 200 - -# tracker sparkline -def get_tracker_sparkline(tracker_uuid, num_day=6): - date_range_sparkline = Date.get_date_range(num_day) - sparklines_value = [] - for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day)) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - sparklines_value.append(int(nb_seen_this_day)) - return sparklines_value - -def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to): - all_item_id = set() +def get_trackers_graph_by_day(l_trackers, num_day=31, date_from=None, date_to=None): if date_from and date_to: - l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True) - if l_date_match: - dict_date_match = dict(l_date_match) - for date_day in Date.substract_date(date_from, date_to): - if date_day in dict_date_match: - all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') - return all_item_id - -def get_tracker_typosquatting_domains(tracker_uuid): - return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}') - -def get_typosquatting_tracked_words_list(): - typosquattings = {} - typos_uuid = get_all_tracker_uuid_by_type("typosquatting") - - for typo_uuid in typos_uuid: - tracker = get_tracker_by_uuid(typo_uuid) - typosquattings[tracker] = get_tracker_typosquatting_domains(typo_uuid) - - return typosquattings - - -def add_tracked_item(tracker_uuid, item_id): - item_date = item_basic.get_item_date(item_id) - # track item - r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) - res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id) - # track nb item by date - if res == 1: - nb_items = r_serv_tracker.zincrby(f'tracker:stat:{tracker_uuid}', 1, int(item_date)) - if nb_items == 1: - update_tracker_daterange(tracker_uuid, item_date) - -def set_tracker_first_seen(tracker_uuid, date): - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date)) - -def set_tracker_last_seen(tracker_uuid, date): - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date)) - -def set_tracker_user_id(tracker_uuid, user_id): - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'user_id', user_id) - -# # TODO: ADD CACHE ??? -def update_tracker_daterange(tracker_uuid, date, op='add'): - date = int(date) - first_seen = get_tracker_first_seen(tracker_uuid) - - if op == 'add': - if not first_seen: - set_tracker_first_seen(tracker_uuid, date) - set_tracker_last_seen(tracker_uuid, date) - else: - first_seen = int(first_seen) - last_seen = int(get_tracker_last_seen(tracker_uuid)) - if date < first_seen: - set_tracker_first_seen(tracker_uuid, date) - if date > last_seen: - set_tracker_last_seen(tracker_uuid, date) - - if op == 'del': - pass - -def remove_tracked_item(item_id): - item_date = item_basic.get_item_date(item_id) - for tracker_uuid in get_item_all_trackers_uuid(item_id): - r_serv_tracker.srem(f'obj:trackers:item:{item_id}', tracker_uuid) - res = r_serv_tracker.srem(f'tracker:item:{tracker_uuid}:{item_date}', item_id) - if res: - r_serv_tracker.zincrby(f'tracker:stat:{tracker_uuid}', -1, int(item_date)) - -def get_item_all_trackers_uuid(obj_id): - #obj_type = 'item' - return r_serv_tracker.smembers(f'obj:trackers:item:{obj_id}') - -def is_obj_tracked(obj_type, subtype, obj_id): - return r_serv_tracker.exists(f'obj:trackers:{obj_type}:{obj_id}') - -def get_obj_all_trackers(obj_type, subtype, obj_id): - return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{obj_id}') - -# # TODO: ADD all Objects + Subtypes -def delete_obj_trackers(obj_type, subtype, id): - if obj_type == 'item': - remove_tracked_item(id) - -def get_email_subject(tracker_uuid): - tracker_description = get_tracker_description(tracker_uuid) - if not tracker_description: - return "AIL framework: Tracker Alert" + date_range = Date.substract_date(date_from, date_to) else: - return 'AIL framework: {}'.format(tracker_description) + date_range = Date.get_date_range(num_day) + list_tracker_stats = [] + for tracker_uuid in l_trackers: + dict_tracker_data = [] + tracker = Tracker(tracker_uuid) + for date_day in date_range: + nb_seen_this_day = tracker.get_nb_objs_by_date(date_day) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)}) + list_tracker_stats.append({"name": tracker.get_tracked(), "Data": dict_tracker_data}) + return list_tracker_stats + +def get_trackers_dashboard(): + trackers = [] + for raw in r_serv_tracker.lrange('trackers:dashboard', 0, -1): + tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) + tracker = Tracker(tracker_uuid) + meta = tracker.get_meta(options={'tags'}) + timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + meta['timestamp'] = timestamp + trackers.append(meta) + return trackers + +def get_user_dashboard(user_id): # TODO SORT + REMOVE OLDER ROWS + trackers = [] + for raw in r_serv_tracker.lrange(f'trackers:user:{user_id}', 0, -1): + tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) + tracker = Tracker(tracker_uuid) + meta = tracker.get_meta(options={'tags'}) + timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + meta['timestamp'] = timestamp + trackers.append(meta) + + return trackers + +def get_trackers_stats(user_id): + stats = {'all': 0} + for tracker_type in get_trackers_types(): + nb_global = get_nb_global_trackers(tracker_type=tracker_type) + nb_user = get_nb_user_trackers(user_id, tracker_type=tracker_type) + stats[tracker_type] = nb_global + nb_user + stats['all'] += nb_global + nb_user + return stats + + + +## Cache ## +# TODO API: Check Tracker type +def trigger_trackers_refresh(tracker_type): + r_cache.set(f'tracker:refresh:{tracker_type}', time.time()) def get_tracker_last_updated_by_type(tracker_type): epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}') if not epoch_update: epoch_update = 0 return float(epoch_update) +# - Cache - # -# # TODO: check type API -def trigger_trackers_refresh(tracker_type): - r_cache.set(f'tracker:refresh:{tracker_type}', time.time()) + + +# Dashboard by user -> tracker + # Add get last tracker in User class + +# Global/User dashboard last trackers + +# -> in ADD function + + + + + + + + + + + + + + + +## Objects ## + +def is_obj_tracked(obj_type, subtype, obj_id): + return r_serv_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') + +def get_obj_trackers(obj_type, subtype, obj_id): + return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') + +def delete_obj_trackers(obj_type, subtype, obj_id): + for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id): + tracker = Tracker(tracker_uuid) + tracker.remove(obj_type, subtype, obj_id) ###################### #### TRACKERS ACL #### -def is_tracker_in_global_level(tracker, tracker_type): - res = r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)) - if res: - for elem_uuid in res: - if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'level')=='1': - return True +## LEVEL ## +def is_tracked_in_global_level(tracked, tracker_type): + for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked): + tracker = Tracker(tracker_uuid) + if tracker.is_level_global(): + return True return False -def is_tracker_in_user_level(tracker, tracker_type, user_id): - res = r_serv_tracker.smembers('user:tracker:{}'.format(user_id)) - if res: - for elem_uuid in res: - if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'tracked')== tracker: - if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'type')== tracker_type: - return True - return False +def is_tracked_in_user_level(tracked, tracker_type, user_id): + trackers_uuid = get_user_trackers_by_tracked(tracker_type, tracked, user_id) + if trackers_uuid: + return True + else: + return False ## API ## def api_check_tracker_uuid(tracker_uuid): @@ -470,6 +722,16 @@ def api_check_tracker_uuid(tracker_uuid): return {"status": "error", "reason": "Unknown uuid"}, 404 return None +def api_check_tracker_acl(tracker_uuid, user_id): + res = api_check_tracker_uuid(tracker_uuid) + if res: + return res + tracker = Tracker(tracker_uuid) + if tracker.is_level_user(): + if tracker.get_user() != user_id or not User(user_id).is_in_role('admin'): + return {"status": "error", "reason": "Access Denied"}, 403 + return None + def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 @@ -483,9 +745,10 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): ##-- ACL --## -#### FIX DB #### +#### FIX DB #### TODO ################################################################### def fix_tracker_stats_per_day(tracker_uuid): - date_from = get_tracker_date(tracker_uuid) + tracker = Tracker(tracker_uuid) + date_from = tracker.get_date() date_to = Date.get_today_date_str() # delete stats r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}') @@ -500,11 +763,12 @@ def fix_tracker_stats_per_day(tracker_uuid): r_serv_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day)) # update first_seen/last_seen - update_tracker_daterange(tracker_uuid, date_day) + tracker.update_daterange(date_day) def fix_tracker_item_link(tracker_uuid): - date_from = get_tracker_first_seen(tracker_uuid) - date_to = get_tracker_last_seen(tracker_uuid) + tracker = Tracker(tracker_uuid) + date_from = tracker.get_first_seen() + date_to = tracker.get_last_seen() if date_from and date_to: for date_day in Date.substract_date(date_from, date_to): @@ -514,11 +778,10 @@ def fix_tracker_item_link(tracker_uuid): def fix_all_tracker_uuid_list(): r_serv_tracker.delete(f'trackers:all') - for tracker_type in get_all_tracker_type(): + for tracker_type in get_trackers_types(): r_serv_tracker.delete(f'trackers:all:{tracker_type}') - l_tracker = get_all_tracker_by_type(tracker_type) - for tracker in l_tracker: - l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type) + for tracked in get_trackers_tracked_by_type(tracker_type): + l_tracker_uuid = get_trackers_by_tracked(tracker_type, tracked) for tracker_uuid in l_tracker_uuid: r_serv_tracker.sadd(f'trackers:all', tracker_uuid) r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) @@ -526,186 +789,60 @@ def fix_all_tracker_uuid_list(): ##-- FIX DB --## #### CREATE TRACKER #### -def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): +def api_validate_tracker_to_add(to_track , tracker_type, nb_words=1): if tracker_type=='regex': - if not is_valid_regex(tracker): + if not is_valid_regex(to_track): return {"status": "error", "reason": "Invalid regex"}, 400 elif tracker_type=='word' or tracker_type=='set': # force lowercase - tracker = tracker.lower() - word_set = set(tracker) + to_track = to_track.lower() + word_set = set(to_track) set_inter = word_set.intersection(special_characters) if set_inter: return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400 - words = tracker.split() + words = to_track.split() # not a word if tracker_type=='word' and len(words)>1: tracker_type = 'set' - # ouput format: tracker1,tracker2,tracker3;2 + # output format: tracker1,tracker2,tracker3;2 if tracker_type=='set': try: nb_words = int(nb_words) - except: + except TypeError: nb_words = 1 - if nb_words==0: + if nb_words == 0: nb_words = 1 words_set = set(words) words_set = sorted(words_set) - if nb_words > len(words_set): nb_words = len(words_set) - tracker = ",".join(words_set) - tracker = "{};{}".format(tracker, nb_words) + to_track = ",".join(words_set) + to_track = f"{to_track};{nb_words}" elif tracker_type == 'typosquatting': - tracker = tracker.lower() + to_track = to_track.lower() # Take only the first term - domain = tracker.split(" ") + domain = to_track.split(" ") if len(domain) > 1: return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400 - if not "." in tracker: + if not "." in to_track: return {"status": "error", "reason": "Invalid domain name"}, 400 - elif tracker_type=='yara_custom': - if not is_valid_yara_rule(tracker): + if not is_valid_yara_rule(to_track): return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif tracker_type=='yara_default': - if not is_valid_default_yara_rule(tracker): + if not is_valid_default_yara_rule(to_track): return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: return {"status": "error", "reason": "Incorrect type"}, 400 - return {"status": "success", "tracker": tracker, "type": tracker_type}, 200 - -def _re_create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard, tracker_uuid, sources, first_seen, last_seen): - create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=dashboard, tracker_uuid=tracker_uuid, sources=sources) - set_tracker_user_id(tracker_uuid, user_id) - if first_seen: - set_tracker_first_seen(tracker_uuid, first_seen) - if last_seen: - set_tracker_last_seen(tracker_uuid, last_seen) - - -def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]): - # edit tracker - if tracker_uuid: - # check if type changed - old_type = get_tracker_type(tracker_uuid) - if not old_type: - edit_tracker = False - else: - edit_tracker = True - old_tracker = get_tracker_by_uuid(tracker_uuid) - old_level = get_tracker_level(tracker_uuid) - tracker_user_id = get_tracker_user_id(tracker_uuid) - - # Create new tracker - else: - edit_tracker = False - # generate tracker uuid - tracker_uuid = str(uuid.uuid4()) - old_type = None - old_tracker = None - - # YARA - if tracker_type == 'yara_custom' or tracker_type == 'yara_default': - # create yara rule - if tracker_type == 'yara_default' and old_type == 'yara': - if not is_default_yara_rule(old_tracker): - filepath = get_yara_rule_file_by_tracker_name(old_tracker) - if filepath: - os.remove(filepath) - tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid) - tracker_type = 'yara' - - elif tracker_type == 'typosquatting': - domain = tracker.split(" ")[0] - typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) - for typo in typo_generation: - r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo) - - # create metadata - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'tracked', tracker) - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'type', tracker_type) - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'date', datetime.date.today().strftime("%Y%m%d")) - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'level', level) - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'dashboard', dashboard) - if not edit_tracker: - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'user_id', user_id) - - if description: - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'description', description) - - if webhook: - r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'webhook', webhook) - - # type change - if edit_tracker: - r_serv_tracker.srem('all:tracker:{}'.format(old_type), old_tracker) - r_serv_tracker.srem('all:tracker_uuid:{}:{}'.format(old_type, old_tracker), tracker_uuid) - if level != old_level: - if level == 0: - r_serv_tracker.srem('global:tracker', tracker_uuid) - elif level == 1: - r_serv_tracker.srem('user:tracker:{}'.format(tracker_user_id), tracker_uuid) - if tracker_type != old_type: - if old_level == 0: - r_serv_tracker.srem('user:tracker:{}:{}'.format(tracker_user_id, old_type), tracker_uuid) - elif old_level == 1: - r_serv_tracker.srem('global:tracker:{}'.format(old_type), tracker_uuid) - if old_type=='yara': - if not is_default_yara_rule(old_tracker): - filepath = get_yara_rule_file_by_tracker_name(old_tracker) - if filepath: - os.remove(filepath) - - # create all tracker set - r_serv_tracker.sadd('all:tracker:{}'.format(tracker_type), tracker) - - # create tracker - uuid map - r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid) - - r_serv_tracker.sadd(f'trackers:all', tracker_uuid) - r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) - - # add display level set - if level == 0: # user only - r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid) - r_serv_tracker.sadd('user:tracker:{}:{}'.format(user_id, tracker_type), tracker_uuid) - elif level == 1: # global - r_serv_tracker.sadd('global:tracker', tracker_uuid) - r_serv_tracker.sadd('global:tracker:{}'.format(tracker_type), tracker_uuid) - - if edit_tracker: - r_serv_tracker.delete(f'tracker:tags:{tracker_uuid}') - r_serv_tracker.delete(f'tracker:mail:{tracker_uuid}') - r_serv_tracker.delete(f'tracker:sources:{tracker_uuid}') - - # create tracker tags list - for tag in tags: - tag = escape(tag) - r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', tag) - Tag.create_custom_tag(tag) - - # create tracker mail notification list - for mail in mails: - r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail)) - - # create tracker sources filter - for source in sources: - # escape source ? - r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source)) - # toggle refresh module tracker list/set - trigger_trackers_refresh(tracker_type) - if tracker_type != old_type: # toggle old type refresh - trigger_trackers_refresh(old_type) - return tracker_uuid + return {"status": "success", "tracked": to_track, "type": tracker_type}, 200 def api_add_tracker(dict_input, user_id): - tracker = dict_input.get('tracker', None) - if not tracker: + to_track = dict_input.get('tracked', None) + if not to_track: return {"status": "error", "reason": "Tracker not provided"}, 400 tracker_type = dict_input.get('type', None) if not tracker_type: @@ -715,10 +852,10 @@ def api_add_tracker(dict_input, user_id): description = escape(description) webhook = dict_input.get('webhook', '') webhook = escape(webhook) - res = api_validate_tracker_to_add(tracker , tracker_type, nb_words=nb_words) + res = api_validate_tracker_to_add(to_track , tracker_type, nb_words=nb_words) if res[1]!=200: return res - tracker = res[0]['tracker'] + to_track = res[0]['tracked'] tracker_type = res[0]['type'] tags = dict_input.get('tags', []) @@ -727,41 +864,142 @@ def api_add_tracker(dict_input, user_id): if res: return res - sources = dict_input.get('sources', []) - res = item_basic.verify_sources_list(sources) - if res: - return res + # Filters # TODO MOVE ME + filters = dict_input.get('filters', {}) + if filters: + if filters.keys() == {'decoded', 'item', 'pgp'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: + filters = {} + for obj_type in filters: + if obj_type not in get_objects_tracked(): + return {"status": "error", "reason": "Invalid Tracker Object type"}, 400 + + if obj_type == 'pgp': + if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: + filters['pgp'].pop('subtypes') + + for filter_name in filters[obj_type]: + if filter_name not in {'mimetypes', 'sources', 'subtypes'}: + return {"status": "error", "reason": "Invalid Filter"}, 400 + elif filter_name == 'mimetypes': # TODO + pass + elif filter_name == 'sources': + if obj_type == 'item': + res = item_basic.verify_sources_list(filters['item']['sources']) + if res: + return res + else: + return {"status": "error", "reason": "Invalid Filter sources"}, 400 + elif filter_name == 'subtypes': + obj_subtypes = set(get_object_all_subtypes(obj_type)) + for subtype in filters[obj_type]['subtypes']: + if subtype not in obj_subtypes: + return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400 - ## TODO: add dashboard key level = dict_input.get('level', 1) try: level = int(level) - if level not in range(0, 1): - level = 1 - except: + except TypeError: + level = 1 + if level not in range(0, 1): level = 1 - tracker_uuid = dict_input.get('uuid', None) - # check edit ACL - if tracker_uuid: - res = api_is_allowed_to_edit_tracker(tracker_uuid, user_id) - if res[1] != 200: - return res - else: - # check if tracker already tracked in global - if level==1: - if is_tracker_in_global_level(tracker, tracker_type) and not tracker_uuid: - return {"status": "error", "reason": "Tracker already exist"}, 409 - else: - if is_tracker_in_user_level(tracker, tracker_type, user_id) and not tracker_uuid: - return {"status": "error", "reason": "Tracker already exist"}, 409 + tracker_uuid = create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters, + tags=tags, mails=mails, webhook=webhook) + + return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200 + +# TODO +def api_edit_tracker(dict_input, user_id): + pass + # tracker_uuid = dict_input.get('uuid', None) + # # check edit ACL + # if tracker_uuid: + # res = api_is_allowed_to_edit_tracker(tracker_uuid, user_id) + # if res[1] != 200: + # return res + # else: + # # check if tracker already tracked in global + # if level==1: + # if is_tracked_in_global_level(to_track, tracker_type) and not tracker_uuid: + # return {"status": "error", "reason": "Tracker already exist"}, 409 + # else: + # if is_tracked_in_user_level(to_track, tracker_type, user_id) and not tracker_uuid: + # return {"status": "error", "reason": "Tracker already exist"}, 409 + +def api_delete_tracker(data, user_id): + tracker_uuid = data.get('uuid') + res = api_check_tracker_acl(tracker_uuid, user_id) + if res: + return res + + tracker = Tracker(tracker_uuid) + return tracker.delete(), 200 + - tracker_uuid = create_tracker(tracker , tracker_type, user_id, level, tags, mails, description, webhook, tracker_uuid=tracker_uuid, sources=sources) - return {'tracker': tracker, 'type': tracker_type, 'uuid': tracker_uuid}, 200 ##-- CREATE TRACKER --## +#################### +#### WORD - SET #### + +def get_words_tracked_list(): # TODO REMOVE ME ???? + return list(r_serv_tracker.smembers('all:tracker:word')) + +def get_tracked_words(): + to_track = {} + for obj_type in get_objects_tracked(): + to_track[obj_type] = _get_tracked_by_obj_type('word', obj_type) + return to_track + +def get_tracked_sets(): + to_track = {} + for obj_type in get_objects_tracked(): + to_track[obj_type] = [] + for tracked in _get_tracked_by_obj_type('set', obj_type): + res = tracked.split(';') + nb_words = int(res[1]) + words_set = res[0].split(',') + to_track[obj_type].append({'words': words_set, 'nb': nb_words, 'tracked': tracked}) + return to_track + +def get_text_word_frequency(content, filtering=True): + content = content.lower() + words_dict = defaultdict(int) + + if filtering: + blob = TextBlob(content, tokenizer=tokenizer) + else: + blob = TextBlob(content) + for word in blob.tokens: + words_dict[word] += 1 + return words_dict + +############### +#### REGEX #### + +def get_tracked_regexs(): + to_track = {} + for obj_type in get_objects_tracked(): + to_track[obj_type] = [] + for tracked in _get_tracked_by_obj_type('regex', obj_type): + to_track[obj_type].append({'regex': re.compile(tracked), 'tracked': tracked}) + return to_track + +######################## +#### TYPO SQUATTING #### + +def get_tracked_typosquatting_domains(tracked): + return r_serv_tracker.smembers(f'tracker:typosquatting:{tracked}') + +def get_tracked_typosquatting(): + to_track = {} + for obj_type in get_objects_tracked(): + to_track[obj_type] = [] + for tracked in _get_tracked_by_obj_type('typosquatting', obj_type): + to_track[obj_type].append({'domains': get_tracked_typosquatting_domains(tracked), 'tracked': tracked}) + return to_track + ############## #### YARA #### def get_yara_rules_dir(): @@ -800,17 +1038,22 @@ def get_all_tracked_yara_files(filter_disabled=False): pass return yara_files -def get_yara_rule_by_uuid(tracker_uuid): - yar_path = get_tracker_by_uuid(tracker_uuid) - return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path)) +def get_tracked_yara_rules(): + to_track = {} + for obj_type in get_objects_tracked(): + rules = {} + for tracked in _get_tracked_by_obj_type('yara', obj_type): + rules[tracked] = os.path.join(get_yara_rules_dir(), tracked) + to_track[obj_type] = yara.compile(filepaths=rules) + print(to_track) + return to_track def reload_yara_rules(): yara_files = get_all_tracked_yara_files() # {uuid: filename} rule_dict = {} for yar_path in yara_files: - l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara') - for tracker_uuid in l_tracker_uuid: + for tracker_uuid in get_trackers_by_tracked('yara', yar_path): rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path) for tracker_uuid in rule_dict: if not os.path.isfile(rule_dict[tracker_uuid]): @@ -819,22 +1062,6 @@ def reload_yara_rules(): rules = yara.compile(filepaths=rule_dict) return rules -# # TODO: -# Avoid useless CHECK -# Empty list == ALL SOURCES -# FIXME MOOVE ME -def get_tracker_sources(tracker, tracker_type): - l_sources = set() - for tracker_uuid in get_tracker_uuid_list(tracker, tracker_type): - sources = get_tracker_uuid_sources(tracker_uuid) - if sources: - for source in get_tracker_uuid_sources(tracker_uuid): - l_sources.add(source) - else: - l_sources = [] - break - return l_sources - def is_valid_yara_rule(yara_rule): try: yara.compile(source=yara_rule) @@ -963,6 +1190,247 @@ def get_yara_rule_content_restapi(request_dict): # date # type +class RetroHunt: + + def __init__(self, task_uuid): + self.uuid = task_uuid + + def exists(self): + return r_serv_tracker.exists(f'tracker:retro_hunt:task:{self.uuid}') + + def _set_field(self, field, value): + return r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', field, value) + + def get_creator(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'creator') + + def get_date(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date') + + def get_date_from(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date_from') + + def get_date_to(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'date_to') + + def get_last_analyzed(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'last') + + def get_name(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'name') + + def get_description(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'description') + + def get_timeout(self): + res = r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'timeout') + if res: + return int(res) + else: + return 30 # # TODO: FIXME use instance limit + + def get_sources(self, r_sort=False): # TODO ADAPT TO ALL OBJECTS ??? + sources = r_serv_tracker.smembers(f'tracker:retro_hunt:task:sources:{self.uuid}') + if not sources: + sources = set(item_basic.get_all_items_sources(filter_dir=False)) + if r_sort: + sources = sorted(sources) + return sources + + def get_tags(self): + return r_serv_tracker.smembers(f'tracker:retro_hunt:task:tags:{self.uuid}') + + def get_mails(self): + return r_serv_tracker.smembers(f'tracker:retro_hunt:task:mails:{self.uuid}') + + def get_state(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'state') + + def _set_state(self, new_state): + curr_state = self.get_state() + if curr_state: + r_serv_tracker.srem(f'tracker:retro_hunt:task:{curr_state}', self.uuid) + r_serv_tracker.sadd(f'tracker:retro_hunt:task:{new_state}', self.uuid) + r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', 'state', new_state) + + def get_rule(self, r_compile=False): + rule = r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'rule') + if r_compile: + rule = os.path.join(get_yara_rules_dir(), rule) + rule_dict = {self.uuid: os.path.join(get_yara_rules_dir(), rule)} + rule = yara.compile(filepaths=rule_dict) + return rule + + # add timeout ? + def get_meta(self, options=set()): + meta = {'uuid': self.uuid, + 'date_from': self.get_date_from(), + 'date_to': self.get_date_to(), + 'name': self.get_name(), + 'state': self.get_state(), + 'rule': self.get_rule(), + } + if 'creator' in options: + meta['creator'] = self.get_creator() + if 'date' in options: + meta['date'] = self.get_date() + if 'description' in options: + meta['description'] = self.get_description() + if 'mails' in options: + meta['mails'] = self.get_mails() + if 'nb_match' in options: + meta['nb_match'] = self.get_nb_match() + if 'progress' in options: + meta['progress'] = self.get_progress() + if 'sources' in options: + meta['progress'] = self.get_sources(r_sort=True) + if 'tags' in options: + meta['tags'] = self.get_tags() + return meta + + def to_pause(self): + to_pause = r_cache.hget(f'tracker:retro_hunt:task:{self.uuid}', 'pause') + if to_pause: + return True + else: + return False + + def pause(self): + self._set_state('paused') + r_cache.hset(f'tracker:retro_hunt:task:{self.uuid}', 'pause', time.time()) + self.clear_cache() + + def resume(self): + r_cache.hdel(f'tracker:retro_hunt:task:{self.uuid}', 'pause') + self._set_state('pending') + + def run(self): # TODO ADD MORE CHECK + self._set_state('running') + + def complete(self): + self._set_state('completed') + self.update_nb_match() + self.clear_cache() + + def get_progress(self): + if self.get_state() == 'completed': + progress = 100 + else: + progress = r_cache.hget(f'tracker:retro_hunt:task:{self.uuid}', 'progress') + if not progress: + progress = self.compute_progress() + return progress + + def compute_progress(self, date_from=None, date_to=None, sources=[], curr_date=None, nb_src_done=0): + # get nb days + if not date_from: + date_from = self.get_date_from() + if not date_to: + date_to = self.get_date_to() + nb_days = Date.get_nb_days_by_daterange(date_from, date_to) + + # nb days completed + if not curr_date: + curr_date = get_retro_hunt_task_current_date(task_uuid) #################################################### + nb_days_done = Date.get_nb_days_by_daterange(date_from, curr_date) - 1 + + # sources + if not sources: + nb_sources = len(self.get_sources()) + else: + nb_sources = len(sources) + + # get progress + progress = ((nb_days_done * nb_sources) + nb_src_done) * 100 / (nb_days * nb_sources) + return int(progress) + + # # TODO: # FIXME: # Cache + + def set_progress(self, progress): + r_cache.hset(f'tracker:retro_hunt:task:{self.uuid}', 'progress', progress) + + def get_nb_match(self): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{self.uuid}', 'nb_match') + + def _set_nb_match(self, nb_match): + r_serv_tracker.hset(f'tracker:retro_hunt:task:{self.uuid}', 'nb_match', nb_match) + + def update_nb_match(self): + l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{self.uuid}', 0, -1, withscores=True) + nb_match = 0 + for row in l_date_value: + nb_match += int(row[1]) + self._set_nb_match(nb_match) + + def clear_cache(self): + r_cache.delete(f'tracker:retro_hunt:task:{self.uuid}') + + def create(self, name, rule, date_from, date_to, creator, description=None, mails=[], tags=[], timeout=30, sources=[], state='pending'): + if self.exists(): + raise Exception('Error: Retro Hunt Task already exists') + + self._set_field('name', escape(name)) + + self._set_field('rule', rule) # TODO FORMAT ??? + + self._set_field('date', datetime.date.today().strftime("%Y%m%d")) + self._set_field('name', escape(name)) + self._set_field('date_from', date_from) + self._set_field('date_to', date_to) + self._set_field('creator', creator) + if description: + self._set_field('description', description) + if timeout: + self._set_field('timeout', int(timeout)) + for source in sources: + r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{self.uuid}', escape(source)) + for tag in tags: + tag = escape(tag) + r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{self.uuid}', tag) + Tag.create_custom_tag(tag) + for mail in mails: + r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{self.uuid}', escape(mail)) + + r_serv_tracker.sadd('tracker:retro_hunt:task:all', self.uuid) + + # add to pending tasks + if state not in ('pending', 'completed', 'paused'): + state = 'pending' + self._set_state(state) + + + # TODO Delete Rule + def delete(self): + if r_serv_tracker.sismember('tracker:retro_hunt:task:running', self.uuid): + return None + + r_serv_tracker.srem('tracker:retro_hunt:task:pending', self.uuid) + r_serv_tracker.delete(f'tracker:retro_hunt:task:{self.uuid}') + r_serv_tracker.delete(f'tracker:retro_hunt:task:sources:{self.uuid}') + r_serv_tracker.delete(f'tracker:retro_hunt:task:tags:{self.uuid}') + r_serv_tracker.delete(f'tracker:retro_hunt:task:mails:{self.uuid}') + + for item_date in get_retro_hunt_all_item_dates(task_uuid): ############################ TODO OBJ ####################### + r_serv_tracker.delete(f'tracker:retro_hunt:task:item:{self.uuid}:{item_date}') + + r_serv_tracker.srem('tracker:retro_hunt:task:all', self.uuid) + r_serv_tracker.srem('tracker:retro_hunt:task:pending', self.uuid) + r_serv_tracker.srem('tracker:retro_hunt:task:paused', self.uuid) + r_serv_tracker.srem('tracker:retro_hunt:task:completed', self.uuid) + + self.clear_cache() + return self.uuid + +def create_retro_hunt(name, rule_type, rule, date_from, date_to, creator, description=None, mails=[], tags=[], timeout=30, sources=[], state='pending', task_uuid=None): + if not task_uuid: + task_uuid = str(uuid.uuid4()) + retro_hunt = RetroHunt(task_uuid) + # rule_type: yara_default - yara custom + rule = save_yara_rule(rule_type, rule, tracker_uuid=retro_hunt.uuid) + retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, + timeout=timeout, sources=sources, state=state) + return retro_hunt.uuid + ## ? ? ? # set tags # set mails @@ -973,94 +1441,41 @@ def get_yara_rule_content_restapi(request_dict): def get_all_retro_hunt_tasks(): return r_serv_tracker.smembers('tracker:retro_hunt:task:all') -def get_all_pending_retro_hunt_tasks(): +def get_retro_hunt_pending_tasks(): return r_serv_tracker.smembers('tracker:retro_hunt:task:pending') -def get_all_running_retro_hunt_tasks(): +def get_retro_hunt_running_tasks(): return r_serv_tracker.smembers('tracker:retro_hunt:task:running') -def get_all_paused_retro_hunt_tasks(): +def get_retro_hunt_paused_tasks(): return r_serv_tracker.smembers('tracker:retro_hunt:task:paused') -## Change STATES ## - -def get_all_completed_retro_hunt_tasks(): +def get_retro_hunt_completed_tasks(): return r_serv_tracker.smembers('tracker:retro_hunt:task:completed') +## Change STATES ## + def get_retro_hunt_task_to_start(): task_uuid = r_serv_tracker.spop('tracker:retro_hunt:task:pending') if task_uuid: - set_retro_hunt_task_state(task_uuid, 'running') + retro_hunt = RetroHunt(task_uuid) + retro_hunt.run() return task_uuid -def pause_retro_hunt_task(task_uuid): - set_retro_hunt_task_state(task_uuid, 'paused') - r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'pause', time.time()) - -def check_retro_hunt_pause(task_uuid): - is_paused = r_cache.hget(f'tracker:retro_hunt:task:{task_uuid}', 'pause') - if is_paused: - return True - else: - return False - -def resume_retro_hunt_task(task_uuid): - r_cache.hdel(f'tracker:retro_hunt:task:{task_uuid}', 'pause') - set_retro_hunt_task_state(task_uuid, 'pending') - ## Metadata ## -def get_retro_hunt_task_name(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'name') +def get_retro_hunt_tasks_metas(): + tasks = [] + for task_uuid in get_all_retro_hunt_tasks(): + retro_hunt = RetroHunt(task_uuid) + tasks.append(retro_hunt.get_meta(options={'date', 'progress', 'nb_match', 'tags'})) + return tasks -def get_retro_hunt_task_state(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'state') -def set_retro_hunt_task_state(task_uuid, new_state): - curr_state = get_retro_hunt_task_state(task_uuid) - if curr_state: - r_serv_tracker.srem(f'tracker:retro_hunt:task:{curr_state}', task_uuid) - r_serv_tracker.sadd(f'tracker:retro_hunt:task:{new_state}', task_uuid) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'state', new_state) -# def get_retro_hunt_task_type(task_uuid): -# return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'type') -# -# def set_retro_hunt_task_type(task_uuid, task_type): -# r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'type', task_type) -# # TODO: yararule -def get_retro_hunt_task_rule(task_uuid, r_compile=False): - #rule_type = 'yara' - rule = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'rule') - if r_compile: - #if rule_type == 'yara' - rule = os.path.join(get_yara_rules_dir(), rule) - rule_dict = {task_uuid : os.path.join(get_yara_rules_dir(), rule)} - rule = yara.compile(filepaths=rule_dict) - return rule -def get_retro_hunt_task_timeout(task_uuid): - res = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'timeout') - if res: - return int(res) - else: - return 30 # # TODO: FIXME use instance limit -def get_retro_hunt_task_date(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date') - -def set_retro_hunt_task_date(task_uuid, date): - return r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date', date) - -def get_retro_hunt_task_date_from(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date_from') - -def get_retro_hunt_task_date_to(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date_to') - -def get_retro_hunt_task_creator(task_uuid): - return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'creator') def get_retro_hunt_last_analyzed(task_uuid): return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'last') @@ -1069,68 +1484,14 @@ def get_retro_hunt_last_analyzed(task_uuid): def set_retro_hunt_last_analyzed(task_uuid, last_id): r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'last', last_id) -def get_retro_hunt_task_sources(task_uuid, r_sort=False): - sources = r_serv_tracker.smembers(f'tracker:retro_hunt:task:sources:{task_uuid}') - if not sources: - sources = set(item_basic.get_all_items_sources(filter_dir=False)) - if r_sort: - sources = sorted(sources) - return sources - -def get_retro_hunt_task_tags(task_uuid): - return r_serv_tracker.smembers(f'tracker:retro_hunt:task:tags:{task_uuid}') - -def get_retro_hunt_task_mails(task_uuid): - return r_serv_tracker.smembers(f'tracker:retro_hunt:task:mails:{task_uuid}') - -# # TODO: ADD TYPE + TIMEOUT -def get_retro_hunt_task_metadata(task_uuid, date=False, progress=False, creator=False, sources=None, tags=None, description=False, nb_match=False): - task_metadata = {'uuid': task_uuid} - task_metadata['state'] = get_retro_hunt_task_state(task_uuid) - task_metadata['name'] = get_retro_hunt_task_name(task_uuid) - task_metadata['rule'] = get_retro_hunt_task_rule(task_uuid) - if creator: - task_metadata['creator'] = get_retro_hunt_task_creator(task_uuid) - if date: - task_metadata['date'] = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date') - task_metadata['date_from'] = get_retro_hunt_task_date_from(task_uuid) - task_metadata['date_to'] = get_retro_hunt_task_date_to(task_uuid) - if description: - task_metadata['description'] = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'description') - if nb_match: - task_metadata['nb_match'] = get_retro_hunt_nb_match(task_uuid) - if progress: - task_metadata['progress'] = get_retro_hunt_task_progress(task_uuid) - if sources: - task_metadata['sources'] = get_retro_hunt_task_sources(task_uuid, r_sort=True) - if tags: - task_metadata['tags'] = get_retro_hunt_task_tags(task_uuid) - return task_metadata - -def get_all_retro_hunt_tasks_with_metadata(): - l_retro_hunt = [] - for task_uuid in get_all_retro_hunt_tasks(): - l_retro_hunt.append(get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, tags=True, nb_match=True)) - return l_retro_hunt - -def get_retro_hunt_task_progress(task_uuid): - if get_retro_hunt_task_state(task_uuid) == 'completed': - progress = 100 - else: - progress = r_cache.hget(f'tracker:retro_hunt:task:{task_uuid}', 'progress') - if not progress: - progress = compute_retro_hunt_task_progress(task_uuid) - return progress - -def set_cache_retro_hunt_task_progress(task_uuid, progress): - r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'progress', progress) +#################################################################################### +#################################################################################### +#################################################################################### +#################################################################################### def set_cache_retro_hunt_task_id(task_uuid, id): r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'id', id) -def clear_retro_hunt_task_cache(task_uuid): - r_cache.delete(f'tracker:retro_hunt:task:{task_uuid}') - # Others # date @@ -1141,82 +1502,31 @@ def clear_retro_hunt_task_cache(task_uuid): # description # state error + +# TODO def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None): - create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, mails=mails, timeout=timeout, description=description, task_uuid=task_uuid, state=state) + retro_hunt = RetroHunt(task_uuid) + retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, + timeout=timeout, sources=sources, state=state) + # TODO if last_id: set_retro_hunt_last_analyzed(task_uuid, last_id) - _set_retro_hunt_nb_match(task_uuid, nb_match) - set_retro_hunt_task_date(task_uuid, date) - - -# # # TODO: TYPE -def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[], tags=[], mails=[], timeout=30, description=None, task_uuid=None, state='pending'): - if not task_uuid: - task_uuid = str(uuid.uuid4()) - - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'name', escape(name)) - - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'rule', rule) - - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date', datetime.date.today().strftime("%Y%m%d")) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date_from', date_from) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date_to', date_to) - - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'creator', creator) - if description: - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'description', description) - if timeout: - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'timeout', int(timeout)) - for source in sources: - r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source)) - for tag in tags: - tag = escape(tag) - r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', tag) - Tag.create_custom_tag(tag) - for mail in mails: - r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail)) - - r_serv_tracker.sadd('tracker:retro_hunt:task:all', task_uuid) - - # add to pending tasks - if state not in ('pending', 'completed', 'paused'): - state = 'pending' - set_retro_hunt_task_state(task_uuid, state) - return task_uuid - -# # TODO: delete rule -def delete_retro_hunt_task(task_uuid): - if r_serv_tracker.sismember('tracker:retro_hunt:task:running', task_uuid): - return None - - r_serv_tracker.srem('tracker:retro_hunt:task:pending', task_uuid) - r_serv_tracker.delete(f'tracker:retro_hunt:task:{task_uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:sources:{task_uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:tags:{task_uuid}') - r_serv_tracker.delete(f'tracker:retro_hunt:task:mails:{task_uuid}') - - for item_date in get_retro_hunt_all_item_dates(task_uuid): - r_serv_tracker.delete(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}') - - r_serv_tracker.srem('tracker:retro_hunt:task:all', task_uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:pending', task_uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:paused', task_uuid) - r_serv_tracker.srem('tracker:retro_hunt:task:completed', task_uuid) - - clear_retro_hunt_task_cache(task_uuid) - return task_uuid + retro_hunt._set_nb_match(nb_match) + retro_hunt._set_field('date', date) def get_retro_hunt_task_current_date(task_uuid): + retro_hunt = RetroHunt(task_uuid) last = get_retro_hunt_last_analyzed(task_uuid) if last: curr_date = item_basic.get_item_date(last) else: - curr_date = get_retro_hunt_task_date_from(task_uuid) + curr_date = retro_hunt.get_date_from() return curr_date def get_retro_hunt_task_nb_src_done(task_uuid, sources=[]): + retro_hunt = RetroHunt(task_uuid) if not sources: - sources = list(get_retro_hunt_task_sources(task_uuid, r_sort=True)) + sources = list(retro_hunt.get_sources(r_sort=True)) else: sources = list(sources) last_id = get_retro_hunt_last_analyzed(task_uuid) @@ -1231,8 +1541,9 @@ def get_retro_hunt_task_nb_src_done(task_uuid, sources=[]): return nb_src_done def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, sources=[]): + retro_hunt = RetroHunt(task_uuid) if not sources: - sources = get_retro_hunt_task_sources(task_uuid, r_sort=True) + sources = retro_hunt.get_sources(r_sort=True) # filter last if filter_last: @@ -1276,31 +1587,6 @@ def get_items_to_analyze(dir, last=None): else: return [] -def compute_retro_hunt_task_progress(task_uuid, date_from=None, date_to=None, sources=[], curr_date=None, nb_src_done=0): - # get nb days - if not date_from: - date_from = get_retro_hunt_task_date_from(task_uuid) - if not date_to: - date_to = get_retro_hunt_task_date_to(task_uuid) - nb_days = Date.get_nb_days_by_daterange(date_from, date_to) - - # nb days completed - if not curr_date: - curr_date = get_retro_hunt_task_current_date(task_uuid) - nb_days_done = Date.get_nb_days_by_daterange(date_from, curr_date) - 1 - - # sources - if not sources: - nb_sources = len(get_retro_hunt_task_sources(task_uuid)) - else: - nb_sources = len(sources) - - # get progress - progress = ((nb_days_done * nb_sources) + nb_src_done) * 100 / (nb_days * nb_sources) - return int(progress) - - # # TODO: # FIXME: # Cache - # # TODO: ADD MAP ID => Retro_Hunt def save_retro_hunt_match(task_uuid, id, object_type='item'): item_date = item_basic.get_item_date(id) @@ -1311,28 +1597,26 @@ def save_retro_hunt_match(task_uuid, id, object_type='item'): # Add map obj_id -> task_uuid r_serv_tracker.sadd(f'obj:retro_hunt:item:{id}', task_uuid) +def delete_retro_hunt_obj(task_uuid, obj_type, obj_id): + item_date = item_basic.get_item_date(obj_id) + res = r_serv_tracker.srem(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}', obj_id) + get_retro_hunt_nb_item_by_day() + # track nb item by date + if res == 1: + r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', -1, int(item_date)) + # Add map obj_id -> task_uuid + r_serv_tracker.srem(f'obj:retro_hunt:item:{obj_id}', task_uuid) + +# TODO +def delete_object_reto_hunts(obj_type, obj_id): + pass +# # get items all retro hunts +# for task_uuid in : ############################################# +# delete_retro_hunt_obj(task_uuid, obj_type, obj_id) + def get_retro_hunt_all_item_dates(task_uuid): return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1) -def get_retro_hunt_nb_match(task_uuid): - nb_match = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match') - if not nb_match: - l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) - nb_match = 0 - for tuple in l_date_value: - nb_match += int(tuple[1]) - return int(nb_match) - -def _set_retro_hunt_nb_match(task_uuid, nb_match): - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match', nb_match) - -def set_retro_hunt_nb_match(task_uuid): - l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) - nb_match = 0 - for tuple in l_date_value: - nb_match += int(tuple[1]) - _set_retro_hunt_nb_match(task_uuid, nb_match) - def get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to): all_item_id = set() if date_from and date_to: @@ -1348,8 +1632,9 @@ def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None): list_stats = [] for task_uuid in l_task_uuid: dict_task_data = [] - retro_name = get_retro_hunt_task_name(task_uuid) - l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) + retro_hunt = RetroHunt(task_uuid) + + l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) ######################## if l_date_match: dict_date_match = dict(l_date_match) if not date_from: @@ -1361,7 +1646,7 @@ def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None): for date_day in date_range: nb_seen_this_day = int(dict_date_match.get(date_day, 0)) dict_task_data.append({"date": date_day,"value": int(nb_seen_this_day)}) - list_stats.append({"name": retro_name,"Data": dict_task_data}) + list_stats.append({"name": retro_hunt.get_name(),"Data": dict_task_data}) return list_stats ## API ## @@ -1378,12 +1663,13 @@ def api_get_retro_hunt_items(dict_input): if res: return res + retro_hunt = RetroHunt(task_uuid) + + # TODO SANITIZE DATES date_from = dict_input.get('date_from', None) date_to = dict_input.get('date_to', None) if date_from is None: - date_from = get_retro_hunt_task_date_from(task_uuid) - if date_from: - date_from = date_from[0] + date_from = retro_hunt.get_date_from() if date_to is None: date_to = date_from if date_from > date_to: @@ -1392,43 +1678,43 @@ def api_get_retro_hunt_items(dict_input): all_items_id = get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to) all_items_id = item_basic.get_all_items_metadata_dict(all_items_id) - res_dict = {} - res_dict['uuid'] = task_uuid - res_dict['date_from'] = date_from - res_dict['date_to'] = date_to - res_dict['items'] = all_items_id + res_dict = {'uuid': task_uuid, + 'date_from': date_from, + 'date_to': date_to, + 'items': all_items_id} return res_dict, 200 def api_pause_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res - task_state = get_retro_hunt_task_state(task_uuid) + retro_hunt = RetroHunt(task_uuid) + task_state = retro_hunt.get_state() if task_state not in ['pending', 'running']: return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400 - pause_retro_hunt_task(task_uuid) + retro_hunt.pause() return task_uuid, 200 def api_resume_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res - task_state = get_retro_hunt_task_state(task_uuid) + retro_hunt = RetroHunt(task_uuid) if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid): - return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400 - resume_retro_hunt_task(task_uuid) + return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400 + retro_hunt.resume() return task_uuid, 200 def api_validate_rule_to_add(rule, rule_type): if rule_type=='yara_custom': if not is_valid_yara_rule(rule): - return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400) + return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif rule_type=='yara_default': if not is_valid_default_yara_rule(rule): - return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400) + return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: - return ({"status": "error", "reason": "Incorrect type"}, 400) - return ({"status": "success", "rule": rule, "type": rule_type}, 200) + return {"status": "error", "reason": "Incorrect type"}, 400 + return {"status": "success", "rule": rule, "type": rule_type}, 200 def api_create_retro_hunt_task(dict_input, creator): # # TODO: API: check mandatory arg @@ -1437,10 +1723,10 @@ def api_create_retro_hunt_task(dict_input, creator): # timeout=30 rule = dict_input.get('rule', None) if not rule: - return ({"status": "error", "reason": "Retro Hunt Rule not provided"}, 400) + return {"status": "error", "reason": "Retro Hunt Rule not provided"}, 400 task_type = dict_input.get('type', None) if not task_type: - return ({"status": "error", "reason": "type not provided"}, 400) + return {"status": "error", "reason": "type not provided"}, 400 # # TODO: limit name = dict_input.get('name', '') @@ -1472,34 +1758,30 @@ def api_create_retro_hunt_task(dict_input, creator): if res: return res - task_uuid = str(uuid.uuid4()) - - # RULE - rule = save_yara_rule(task_type, rule, tracker_uuid=task_uuid) - task_type = 'yara' - - task_uuid = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, - tags=tags, mails=mails, timeout=30, description=description, task_uuid=task_uuid) - - return ({'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200) + task_uuid = create_retro_hunt(name, task_type, rule, date_from, date_to, creator, description=description, + mails=mails, tags=tags, timeout=30, sources=sources) + return {'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200 def api_delete_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res if r_serv_tracker.sismember('tracker:retro_hunt:task:running', task_uuid): - return ({"status": "error", "reason": "You can't delete a running task"}, 400) + return {"status": "error", "reason": "You can't delete a running task"}, 400 else: - return (delete_retro_hunt_task(task_uuid), 200) + retro_hunt = RetroHunt(task_uuid) + return retro_hunt.delete(), 200 #### DB FIX #### def get_trackers_tags(): tags = set() - for tracker_uuid in get_all_tracker_uuid(): - for tag in get_tracker_tags(tracker_uuid): + for tracker_uuid in get_trackers(): + tracker = Tracker(tracker_uuid) + for tag in tracker.get_tags(): tags.add(tag) for task_uuid in get_all_retro_hunt_tasks(): - for tag in get_retro_hunt_task_tags(task_uuid): + retro_hunt = RetroHunt(task_uuid) + for tag in retro_hunt.get_tags(): tags.add(tag) return tags diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index aa270b2b..4d8aef9f 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -40,6 +40,9 @@ def get_object_all_subtypes(obj_type): return ['telegram', 'twitter', 'jabber'] return [] +def get_objects_tracked(): + return ['decoded', 'item', 'pgp'] + def get_all_objects_with_subtypes_tuple(): str_objs = [] for obj_type in get_all_objects(): diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 8ad532aa..5a111eb9 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -95,17 +95,18 @@ def convert_byte_offset_to_string(b_content, offset): def get_tracker_match(obj_id, content): extracted = [] extracted_yara = [] - trackers = Tracker.get_obj_all_trackers('item', '', obj_id) + trackers = Tracker.get_obj_trackers('item', '', obj_id) for tracker_uuid in trackers: - tracker_type = Tracker.get_tracker_type(tracker_uuid) + tracker = Tracker.Tracker(tracker_uuid) + tracker_type = tracker.get_type() # print(tracker_type) - tracker = Tracker.get_tracker_by_uuid(tracker_uuid) + tracked = tracker.get_tracked() if tracker_type == 'regex': # TODO Improve word detection -> word delimiter - regex_match = regex_helper.regex_finditer(r_key, tracker, obj_id, content) + regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content) for match in regex_match: - extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}']) + extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}']) elif tracker_type == 'yara': - rule = Tracker.get_yara_rule_by_uuid(tracker_uuid) + rule = tracker.get_rule() rule.match(data=content.encode(), callback=_get_yara_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=30) yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}') @@ -113,20 +114,20 @@ def get_tracker_match(obj_id, content): extracted = [] for match in yara_match: start, end, value = match.split(':', 2) - extracted_yara.append([int(start), int(end), value, f'tracker:{tracker_uuid}']) + extracted_yara.append([int(start), int(end), value, f'tracker:{tracker.uuid}']) elif tracker_type == 'word' or tracker_type == 'set': if tracker_type == 'set': - tracker = tracker.rsplit(';', 1)[0] - words = tracker.split(',') + tracked = tracked.rsplit(';', 1)[0] + words = tracked.split(',') else: - words = [tracker] + words = [tracked] for word in words: regex = _get_word_regex(word) regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content) # print(regex_match) for match in regex_match: - extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}']) + extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}']) # Convert byte offset to string offset if extracted_yara: diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index 96073ed1..be1e1463 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -2,12 +2,13 @@ # -*-coding:UTF-8 -* import os +import re import sys from flask import url_for from hashlib import sha256 -from pymisp import MISPObject, MISPAttribute +from pymisp import MISPObject sys.path.append(os.environ['AIL_BIN']) ################################## @@ -175,31 +176,57 @@ def get_all_cryptocurrencies(): cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype) return cryptos - def get_all_cryptocurrencies_by_subtype(subtype): return get_all_id('cryptocurrency', subtype) +def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail + if subtype == '': + pass + elif subtype == 'name': + pass + elif subtype == 'mail': + pass + return name_to_search -# TODO save object -def import_misp_object(misp_obj): - """ - :type misp_obj: MISPObject - """ - obj_id = None - obj_subtype = None - for attribute in misp_obj.attributes: - if attribute.object_relation == 'address': # TODO: handle xmr address field - obj_id = attribute.value - elif attribute.object_relation == 'symbol': - obj_subtype = get_subtype_by_symbol(attribute.value) - if obj_id and obj_subtype: - obj = CryptoCurrency(obj_id, obj_subtype) - first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj) - tags = obj.get_misp_object_tags(misp_obj) - # for tag in tags: - # obj.add_tag() +def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False): + cryptocurrencies = {} + # for subtype in subtypes: + r_name = sanitize_cryptocurrency_name_to_search(name_to_search, subtype) + if not name_to_search or isinstance(r_name, dict): + # break + return cryptocurrencies + r_name = re.compile(r_name) + for crypto_name in get_all_cryptocurrencies_by_subtype(subtype): + res = re.search(r_name, crypto_name) + if res: + cryptocurrencies[crypto_name] = {} + if r_pos: + cryptocurrencies[crypto_name]['hl-start'] = res.start() + cryptocurrencies[crypto_name]['hl-end'] = res.end() + return cryptocurrencies -if __name__ == '__main__': - res = get_all_cryptocurrencies() - print(res) +# # TODO save object +# def import_misp_object(misp_obj): +# """ +# :type misp_obj: MISPObject +# """ +# obj_id = None +# obj_subtype = None +# for attribute in misp_obj.attributes: +# if attribute.object_relation == 'address': # TODO: handle xmr address field +# obj_id = attribute.value +# elif attribute.object_relation == 'symbol': +# obj_subtype = get_subtype_by_symbol(attribute.value) +# if obj_id and obj_subtype: +# obj = CryptoCurrency(obj_id, obj_subtype) +# first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj) +# tags = obj.get_misp_object_tags(misp_obj) +# # for tag in tags: +# # obj.add_tag() + + +# if __name__ == '__main__': +# name_to_search = '3c' +# subtype = 'bitcoin' +# print(search_cryptocurrency_by_name(name_to_search, subtype)) diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index 6c84675c..e0315d58 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -2,9 +2,11 @@ # -*-coding:UTF-8 -* import os +import re import sys from flask import url_for + from pymisp import MISPObject import requests @@ -110,6 +112,25 @@ def get_cves_meta(cves_id, options=set()): dict_cve[cve_id] = cve.get_meta(options=options) return dict_cve +def sanitize_cve_name_to_search(name_to_search): # TODO FILTER NAME + return name_to_search + +def search_cves_by_name(name_to_search, r_pos=False): + cves = {} + # for subtype in subtypes: + r_name = sanitize_cve_name_to_search(name_to_search) + if not name_to_search or isinstance(r_name, dict): + return cves + r_name = re.compile(r_name) + for cve_name in get_all_cves(): + res = re.search(r_name, cve_name) + if res: + cves[cve_name] = {} + if r_pos: + cves[cve_name]['hl-start'] = res.start() + cves[cve_name]['hl-end'] = res.end() + return cves + def api_get_cves_range_by_daterange(date_from, date_to): cves = [] for date in Date.substract_date(date_from, date_to): @@ -133,3 +154,5 @@ def get_cve_graphline(cve_id): # if __name__ == '__main__': +# name_to_search = '98' +# print(search_cves_by_name(name_to_search)) \ No newline at end of file diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 0f5a5a31..f1846da8 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -2,6 +2,7 @@ # -*-coding:UTF-8 -* import os +import re import sys import magic import requests @@ -9,6 +10,7 @@ import zipfile from flask import url_for from io import BytesIO + from pymisp import MISPObject sys.path.append(os.environ['AIL_BIN']) @@ -114,11 +116,15 @@ class Decoded(AbstractDaterangeObject): def get_filepath(self, mimetype=None): return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype)) - def get_content(self, mimetype=None): + def get_content(self, mimetype=None, r_str=False): filepath = self.get_filepath(mimetype=mimetype) - with open(filepath, 'rb') as f: - file_content = BytesIO(f.read()) - return file_content + if r_str: + with open(filepath, 'r') as f: + content = f.read() + else: + with open(filepath, 'rb') as f: + content = BytesIO(f.read()) + return content def get_zip_content(self): # mimetype = self.get_estimated_type() @@ -347,6 +353,25 @@ def sanitise_mimetype(mimetype): else: return None +def sanitize_decoded_name_to_search(name_to_search): # TODO FILTER NAME + return name_to_search + +def search_decodeds_by_name(name_to_search, r_pos=False): + decodeds = {} + # for subtype in subtypes: + r_name = sanitize_decoded_name_to_search(name_to_search) + if not name_to_search or isinstance(r_name, dict): + return decodeds + r_name = re.compile(r_name) + for decoded_name in get_all_decodeds(): + res = re.search(r_name, decoded_name) + if res: + decodeds[decoded_name] = {} + if r_pos: + decodeds[decoded_name]['hl-start'] = res.start() + decodeds[decoded_name]['hl-end'] = res.end() + return decodeds + ############################################################################ def sanityze_decoder_names(decoder_name): @@ -512,4 +537,7 @@ def get_all_decodeds_files(): decodeds.append(file) return decodeds + # if __name__ == '__main__': +# name_to_search = '4d36' +# print(search_decodeds_by_name(name_to_search)) \ No newline at end of file diff --git a/bin/lib/objects/Domains.py b/bin/lib/objects/Domains.py index 8b3e40f7..914ff942 100755 --- a/bin/lib/objects/Domains.py +++ b/bin/lib/objects/Domains.py @@ -96,6 +96,9 @@ class Domain(AbstractObject): elif int(last_check) < date: self._set_last_check(date) + def get_content(self): + return self.id + def get_last_origin(self, obj=False): origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')} if obj and origin['item']: diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index d28654ca..f107ca6c 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -79,7 +79,7 @@ class Item(AbstractObject): else: return filename - def get_content(self, binary=False): + def get_content(self, r_str=True, binary=False): """ Returns Item content """ diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index 201b7226..fd7297b9 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -2,6 +2,7 @@ # -*-coding:UTF-8 -* import os +import re import sys from flask import url_for @@ -100,5 +101,35 @@ def get_all_pgps(): def get_all_pgps_by_subtype(subtype): return get_all_id('pgp', subtype) +# TODO FILTER NAME + Key + mail +def sanitize_pgp_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail + if subtype == 'key': + pass + elif subtype == 'name': + pass + elif subtype == 'mail': + pass + return name_to_search + +def search_pgps_by_name(name_to_search, subtype, r_pos=False): + pgps = {} + # for subtype in subtypes: + r_name = sanitize_pgp_name_to_search(name_to_search, subtype) + if not name_to_search or isinstance(r_name, dict): + # break + return pgps + r_name = re.compile(r_name) + for pgp_name in get_all_pgps_by_subtype(subtype): + res = re.search(r_name, pgp_name) + if res: + pgps[pgp_name] = {} + if r_pos: + pgps[pgp_name]['hl-start'] = res.start() + pgps[pgp_name]['hl-end'] = res.end() + return pgps + # if __name__ == '__main__': +# name_to_search = 'ex' +# subtype = 'name' +# print(search_pgps_by_name(name_to_search, subtype)) diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index bf1a92d0..68814ab8 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -3,6 +3,7 @@ import base64 import os +import re import sys from hashlib import sha256 @@ -71,6 +72,9 @@ class Screenshot(AbstractObject): file_content = BytesIO(f.read()) return file_content + def get_content(self): + return self.get_file_content() + def get_misp_object(self): obj_attrs = [] obj = MISPObject('file') @@ -129,4 +133,26 @@ def create_screenshot(content, size_limit=5000000, b64=True, force=False): return screenshot return None -#if __name__ == '__main__': +def sanitize_screenshot_name_to_search(name_to_search): # TODO FILTER NAME + return name_to_search + +def search_screenshots_by_name(name_to_search, r_pos=False): + screenshots = {} + # for subtype in subtypes: + r_name = sanitize_screenshot_name_to_search(name_to_search) + if not name_to_search or isinstance(r_name, dict): + return screenshots + r_name = re.compile(r_name) + for screenshot_name in get_all_screenshots(): + res = re.search(r_name, screenshot_name) + if res: + screenshots[screenshot_name] = {} + if r_pos: + screenshots[screenshot_name]['hl-start'] = res.start() + screenshots[screenshot_name]['hl-end'] = res.end() + return screenshots + + +# if __name__ == '__main__': +# name_to_search = '29ba' +# print(search_screenshots_by_name(name_to_search)) diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index b4226d28..8770cfc1 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -3,13 +3,11 @@ import os import sys -import redis +import re from flask import url_for from pymisp import MISPObject -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) - sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages @@ -61,7 +59,7 @@ class Username(AbstractSubtypeObject): else: style = 'fas' icon = '\uf007' - return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} def get_meta(self, options=set()): meta = self._get_meta(options=options) @@ -74,15 +72,15 @@ class Username(AbstractSubtypeObject): obj_attrs = [] if self.subtype == 'telegram': obj = MISPObject('telegram-account', standalone=True) - obj_attrs.append( obj.add_attribute('username', value=self.id) ) + obj_attrs.append(obj.add_attribute('username', value=self.id)) elif self.subtype == 'twitter': obj = MISPObject('twitter-account', standalone=True) - obj_attrs.append( obj.add_attribute('name', value=self.id) ) + obj_attrs.append(obj.add_attribute('name', value=self.id)) else: obj = MISPObject('user-account', standalone=True) - obj_attrs.append( obj.add_attribute('username', value=self.id) ) + obj_attrs.append(obj.add_attribute('username', value=self.id)) obj.first_seen = self.get_first_seen() obj.last_seen = self.get_last_seen() @@ -107,9 +105,30 @@ def get_all_usernames(): def get_all_usernames_by_subtype(subtype): return get_all_id('username', subtype) +# TODO FILTER NAME + Key + mail +def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME + + return name_to_search + +def search_usernames_by_name(name_to_search, subtype, r_pos=False): + usernames = {} + # for subtype in subtypes: + r_name = sanitize_username_name_to_search(name_to_search, subtype) + if not name_to_search or isinstance(r_name, dict): + # break + return usernames + r_name = re.compile(r_name) + for user_name in get_all_usernames_by_subtype(subtype): + res = re.search(r_name, user_name) + if res: + usernames[user_name] = {} + if r_pos: + usernames[user_name]['hl-start'] = res.start() + usernames[user_name]['hl-end'] = res.end() + return usernames -if __name__ == '__main__': - - obj = Username('ninechantw', 'telegram') - print(obj.get_misp_object().to_json()) +# if __name__ == '__main__': +# name_to_search = 'co' +# subtype = 'telegram' +# print(search_usernames_by_name(name_to_search, subtype)) diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index b2b4c6f0..6307d3c4 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -108,6 +108,9 @@ class AbstractDaterangeObject(AbstractObject, ABC): sparkline.append(self.get_nb_seen_by_date(date)) return sparkline + def get_content(self): + return self.id + def _add_create(self): r_object.sadd(f'{self.type}:all', self.id) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index c10f0acd..b65c9ace 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -19,9 +19,9 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import Tag from lib import Duplicate -from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type +from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations -from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers +from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers class AbstractObject(ABC): @@ -85,6 +85,13 @@ class AbstractObject(ABC): #- Tags -# + @abstractmethod + def get_content(self): + """ + Get Object Content + """ + pass + ## Duplicates ## def get_duplicates(self): return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id) @@ -125,7 +132,7 @@ class AbstractObject(ABC): return is_obj_tracked(self.type, self.subtype, self.id) def get_trackers(self): - return get_obj_all_trackers(self.type, self.subtype, self.id) + return get_obj_trackers(self.type, self.subtype, self.id) def delete_trackers(self): return delete_obj_trackers(self.type, self.subtype, self.id) @@ -134,13 +141,14 @@ class AbstractObject(ABC): def _delete(self): # DELETE TAGS - Tag.delete_obj_all_tags(self.id, self.type) # ########### # TODO: # TODO: # FIXME: + Tag.delete_object_tags(self.type, self.get_subtype(r_str=True), self.id) # remove from tracker self.delete_trackers() + # remove from retro hunt currently item only TODO # remove from investigations self.delete_investigations() - - # # TODO: remove from correlation + # Delete Correlations + delete_obj_correlations(self.type, self.get_subtype(r_str=True), self.id) @abstractmethod def delete(self): diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index bc466473..690c8b81 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -116,6 +116,9 @@ class AbstractSubtypeObject(AbstractObject, ABC): if date > last_seen: self.set_last_seen(date) + def get_content(self): + return self.id + def get_sparkline(self): sparkline = [] for date in Date.get_previous_date_list(6): diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index c9e1d7bc..0a1e2f2c 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -138,9 +138,14 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False): def get_objects_meta(objs, options=set(), flask_context=False): metas = [] - for obj_dict in objs: - metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options, - flask_context=flask_context)) + for obj in objs: + if isinstance(obj, dict): + obj_type = obj['type'] + subtype = obj['subtype'] + obj_id = obj['id'] + else: + obj_type, subtype, obj_id = obj.split(':', 2) + metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context)) return metas @@ -162,7 +167,25 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): return meta -def get_ui_obj_tag_table_keys(obj_type): +#### OBJ FILTERS #### + +def is_filtered(obj, filters): + if 'mimetypes' in filters: + mimetype = obj.get_mimetype() + if mimetype not in filters['mimetypes']: + return True + if 'sources' in filters: + obj_source = obj.get_source() + if obj_source not in filters['sources']: + return True + if 'subtypes' in filters: + subtype = obj.get_subtype(r_str=True) + if subtype not in filters['subtypes']: + return True + return False + + +def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ Warning: use only in flask (dynamic templates) """ @@ -170,6 +193,7 @@ def get_ui_obj_tag_table_keys(obj_type): return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot + # # # # MISP OBJECTS # # # # # # TODO: CHECK IF object already have an UUID @@ -265,6 +289,7 @@ def api_sanitize_object_type(obj_type): if not is_valid_object_type(obj_type): return {'status': 'error', 'reason': 'Incorrect object type'}, 400 +#### CORRELATION #### def get_obj_correlations(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) @@ -292,13 +317,14 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) + # print(objs) for obj_tuple in objs: obj1_type, subtype1, id1 = obj_tuple add_obj_tags(obj1_type, subtype1, id1, tags) return objs ################################################################################ -################################################################################ +################################################################################ TODO ################################################################################ def delete_obj_correlations(obj_type, subtype, obj_id): @@ -357,7 +383,7 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_ "links": create_correlation_graph_links(links)} -############### +# --- CORRELATION --- # # if __name__ == '__main__': diff --git a/bin/modules/Decoder.py b/bin/modules/Decoder.py index 41a3fae1..e94f192e 100755 --- a/bin/modules/Decoder.py +++ b/bin/modules/Decoder.py @@ -23,6 +23,9 @@ from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader from lib.objects.Items import Item from lib.objects.Decodeds import Decoded +from trackers.Tracker_Term import Tracker_Term +from trackers.Tracker_Regex import Tracker_Regex +from trackers.Tracker_Yara import Tracker_Yara config_loader = ConfigLoader() hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal") @@ -76,6 +79,10 @@ class Decoder(AbstractModule): # Waiting time in seconds between to message processed self.pending_seconds = 1 + self.tracker_term = Tracker_Term(queue=False) + self.tracker_regex = Tracker_Regex(queue=False) + self.tracker_yara = Tracker_Yara(queue=False) + # Send module state to logs self.redis_logger.info(f'Module {self.module_name} initialized') @@ -84,6 +91,7 @@ class Decoder(AbstractModule): item = Item(message) content = item.get_content() date = item.get_date() + new_decodeds = [] for decoder in self.decoder_order: find = False @@ -96,8 +104,8 @@ class Decoder(AbstractModule): encodeds = set(encodeds) for encoded in encodeds: - find = False if len(encoded) >= decoder['encoded_min_size']: + find = True decoded_file = self.decoder_function[dname](encoded) sha1_string = sha1(decoded_file).hexdigest() @@ -109,6 +117,7 @@ class Decoder(AbstractModule): print(sha1_string, item.id) raise Exception(f'Invalid mimetype: {decoded.id} {item.id}') decoded.save_file(decoded_file, mimetype) + new_decodeds.append(decoded.id) else: mimetype = decoded.get_mimetype() decoded.add(dname, date, item.id, mimetype=mimetype) @@ -125,6 +134,13 @@ class Decoder(AbstractModule): msg = f'infoleak:automatic-detection="{dname}";{item.id}' self.add_message_to_queue(msg, 'Tags') + #################### + # TRACKERS DECODED + for decoded_id in new_decodeds: + self.tracker_term.compute(decoded_id, obj_type='decoded') + self.tracker_regex.compute(decoded_id, obj_type='decoded') + self.tracker_yara.compute(decoded_id, obj_type='decoded') + if __name__ == '__main__': module = Decoder() diff --git a/bin/modules/PgpDump.py b/bin/modules/PgpDump.py index 45982260..aa4b629b 100755 --- a/bin/modules/PgpDump.py +++ b/bin/modules/PgpDump.py @@ -216,16 +216,16 @@ class PgpDump(AbstractModule): pgp = Pgps.Pgp(name, 'name') pgp.add(date, self.item_id) print(f' name: {name}') - self.tracker_term.compute(self.item_id, item_content=name) - self.tracker_regex.compute(self.item_id, content=name) - self.tracker_yara.compute(self.item_id, item_content=name) + self.tracker_term.compute(name, obj_type='pgp', subtype='name') + self.tracker_regex.compute(name, obj_type='pgp', subtype='name') + self.tracker_yara.compute(name, obj_type='pgp', subtype='name') for mail in self.mails: pgp = Pgps.Pgp(mail, 'mail') pgp.add(date, self.item_id) print(f' mail: {mail}') - self.tracker_term.compute(self.item_id, item_content=mail) - self.tracker_regex.compute(self.item_id, content=mail) - self.tracker_yara.compute(self.item_id, item_content=mail) + self.tracker_term.compute(mail, obj_type='pgp', subtype='mail') + self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail') + self.tracker_yara.compute(mail, obj_type='pgp', subtype='mail') # Keys extracted from PGP PRIVATE KEY BLOCK for key in self.private_keys: diff --git a/bin/packages/Term.py b/bin/packages/Term.py deleted file mode 100755 index bc4fced0..00000000 --- a/bin/packages/Term.py +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import re -import sys -import time -import uuid -import datetime - -from collections import defaultdict -from flask import escape - -from nltk.tokenize import RegexpTokenizer -from textblob import TextBlob - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ConfigLoader -from lib import Tracker -from packages import Date -from lib.objects import Items - -config_loader = ConfigLoader.ConfigLoader() -r_serv_term = config_loader.get_db_conn("Kvrocks_Trackers") -config_loader = None - -email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' -email_regex = re.compile(email_regex) - -special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\') -special_characters.add('\\s') - -# NLTK tokenizer -tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+', - gaps=True, discard_empty=True) - -def is_valid_uuid_v4(UUID): - if not UUID: - return False - UUID = UUID.replace('-', '') - try: - uuid_test = uuid.UUID(hex=UUID, version=4) - return uuid_test.hex == UUID - except: - return False - -# # TODO: use new package => duplicate fct -def is_in_role(user_id, role): - if r_serv_term.sismember('user_role:{}'.format(role), user_id): - return True - else: - return False - -def check_term_uuid_valid_access(term_uuid, user_id): - if not is_valid_uuid_v4(term_uuid): - return {"status": "error", "reason": "Invalid uuid"}, 400 - level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') - if not level: - return {"status": "error", "reason": "Unknown uuid"}, 404 - if level == 0: - if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id: - if not is_in_role(user_id, 'admin'): - return {"status": "error", "reason": "Unknown uuid"}, 404 - return None - - -def is_valid_mail(email): - result = email_regex.match(email) - if result: - return True - else: - return False - -def verify_mail_list(mail_list): - for mail in mail_list: - if not is_valid_mail(mail): - return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400 - return None - -def is_valid_regex(term_regex): - try: - re.compile(term_regex) - return True - except: - return False - -def get_text_word_frequency(item_content, filtering=True): - item_content = item_content.lower() - words_dict = defaultdict(int) - - if filtering: - blob = TextBlob(item_content, tokenizer=tokenizer) - else: - blob = TextBlob(item_content) - for word in blob.tokens: - words_dict[word] += 1 - return words_dict - -# # TODO: create all tracked words -def get_tracked_words_list(): - return list(r_serv_term.smembers('all:tracker:word')) - -def get_set_tracked_words_list(): - set_list = r_serv_term.smembers('all:tracker:set') - all_set_list = [] - for elem in set_list: - res = elem.split(';') - num_words = int(res[1]) - ter_set = res[0].split(',') - all_set_list.append((ter_set, num_words, elem)) - return all_set_list - -def get_regex_tracked_words_dict(): - regex_list = r_serv_term.smembers('all:tracker:regex') - dict_tracked_regex = {} - for regex in regex_list: - dict_tracked_regex[regex] = re.compile(regex) - return dict_tracked_regex - -def get_tracked_term_list_item(term_uuid, date_from, date_to): - all_item_id = [] - if date_from and date_to: - for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)): - all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date))) - return all_item_id - -def is_term_tracked_in_global_level(term, term_type): - res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)) - if res: - for elem_uuid in res: - if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level') == '1': - return True - return False - -def is_term_tracked_in_user_level(term, term_type, user_id): - res = r_serv_term.smembers('user:tracker:{}'.format(user_id)) - if res: - for elem_uuid in res: - if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked') == term: - if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type') == term_type: - return True - return False - -def parse_json_term_to_add(dict_input, user_id): - term = dict_input.get('term', None) - if not term: - return {"status": "error", "reason": "Term not provided"}, 400 - term_type = dict_input.get('type', None) - if not term_type: - return {"status": "error", "reason": "Term type not provided"}, 400 - - nb_words = dict_input.get('nb_words', 1) - - description = dict_input.get('description', '') - description = escape(description) - - webhook = dict_input.get('webhook', '') - webhook = escape(webhook) - - res = parse_tracked_term_to_add(term, term_type, nb_words=nb_words) - if res[1] != 200: - return res - term = res[0]['term'] - term_type = res[0]['type'] - - tags = dict_input.get('tags', []) - mails = dict_input.get('mails', []) - res = verify_mail_list(mails) - if res: - return res - - level = dict_input.get('level', 1) - try: - level = int(level) - if level not in range(0, 1): - level = 1 - except: - level = 1 - - # check if term already tracked in global - if level == 1: - if is_term_tracked_in_global_level(term, term_type): - return {"status": "error", "reason": "Term already tracked"}, 409 - else: - if is_term_tracked_in_user_level(term, term_type, user_id): - return {"status": "error", "reason": "Term already tracked"}, 409 - - term_uuid = add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook) - - return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200 - - -def parse_tracked_term_to_add(term, term_type, nb_words=1): - if term_type == 'regex': - if not is_valid_regex(term): - return {"status": "error", "reason": "Invalid regex"}, 400 - elif term_type == 'word' or term_type == 'set': - # force lowercase - term = term.lower() - word_set = set(term) - set_inter = word_set.intersection(special_characters) - if set_inter: - return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400 - words = term.split() - # not a word - if term_type == 'word' and len(words) > 1: - term_type = 'set' - - # output format: term1,term2,term3;2 - if term_type == 'set': - try: - nb_words = int(nb_words) - except: - nb_words = 1 - if nb_words == 0: - nb_words = 1 - - words_set = set(words) - words_set = sorted(words_set) - - if nb_words > len(words_set): - nb_words = len(words_set) - - term = ",".join(words_set) - term = "{};{}".format(term, nb_words) - - elif term_type == 'yara_custom': - if not Tracker.is_valid_yara_rule(term): - return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 - elif term_type == 'yara_default': - if not Tracker.is_valid_default_yara_rule(term): - return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 - else: - return {"status": "error", "reason": "Incorrect type"}, 400 - return {"status": "success", "term": term, "type": term_type}, 200 - -def add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook, dashboard=0): - - term_uuid = str(uuid.uuid4()) - - # YARA - if term_type == 'yara_custom' or term_type == 'yara_default': - term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid) - term_type = 'yara' - - # create metadata - r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked', term) # # TODO: use hash - r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type) - r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d")) - r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id) - r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level) - r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard) - - if description: - r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description) - - if webhook: - r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook) - - # create all term set - r_serv_term.sadd('all:tracker:{}'.format(term_type), term) - - # create term - uuid map - r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) - - # add display level set - if level == 0: # user only - r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid) - r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid) - elif level == 1: # global - r_serv_term.sadd('global:tracker', term_uuid) - r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid) - - # create term tags list - for tag in tags: - r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), escape(tag)) - - # create term tags mail notification list - for mail in mails: - r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), escape(mail)) - - # toggle refresh module tracker list/set - r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time()) - - return term_uuid - - -def parse_tracked_term_to_delete(dict_input, user_id): - term_uuid = dict_input.get("uuid", None) - res = check_term_uuid_valid_access(term_uuid, user_id) - if res: - return res - delete_term(term_uuid) - return {"uuid": term_uuid}, 200 - - -# # TODO: MOVE IN TRACKER -def delete_term(term_uuid): - term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') - term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') - level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') - r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) - - r_serv_term.srem(f'trackers:all', term_uuid) - r_serv_term.srem(f'trackers:all:{term_type}', term_uuid) - - # Term not tracked by other users - if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)): - r_serv_term.srem('all:tracker:{}'.format(term_type), term) - - # toggle refresh module tracker list/set - r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time()) - - if level == '0': # user only - user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') - r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid) - r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid) - elif level == '1': # global - r_serv_term.srem('global:tracker', term_uuid) - r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid) - - # delete metatadata - r_serv_term.delete('tracker:{}'.format(term_uuid)) - - # remove tags - r_serv_term.delete('tracker:tags:{}'.format(term_uuid)) - - # remove mails - r_serv_term.delete('tracker:mail:{}'.format(term_uuid)) - - # remove sources - r_serv_term.delete('tracker:sources:{}'.format(term_uuid)) - - # remove item set - all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True) - if all_item_date: - all_item_date = dict(all_item_date) - for date in all_item_date: - for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'): - r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid) - r_serv_term.delete(f'tracker:item:{term_uuid}:{date}') - r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) - - if term_type == 'yara': - # delete custom rule - if not Tracker.is_default_yara_rule(term): - filepath = Tracker.get_yara_rule_file_by_tracker_name(term) - if filepath: - os.remove(filepath) - -def replace_tracker_description(term_uuid, description): - description = escape(description) - r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description) - -def replace_tracked_term_tags(term_uuid, tags): - r_serv_term.delete('tracker:tags:{}'.format(term_uuid)) - for tag in tags: - tag = escape(tag) - r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag) - -def replace_tracked_term_mails(term_uuid, mails): - res = verify_mail_list(mails) - if res: - return res - else: - r_serv_term.delete('tracker:mail:{}'.format(term_uuid)) - for mail in mails: - mail = escape(mail) - r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail) - -def get_term_uuid_list(term, term_type): - return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))) - -def get_term_tags(term_uuid): - return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))) - -def get_term_mails(term_uuid): - return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))) - -def get_term_webhook(term_uuid): - return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook") - -def add_tracked_item(term_uuid, item_id, item_date): - # track item - r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id) - # track nb item by date - r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), {item_date: item_date}) - -def create_token_statistics(item_date, word, nb): - r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), 1, word) - r_serv_term.zincrby('stat_token_total_by_day:{}'.format(item_date), nb, word) - r_serv_term.sadd('stat_token_history', item_date) - -def delete_token_statistics_by_date(item_date): - r_serv_term.delete('stat_token_per_item_by_day:{}'.format(item_date)) - r_serv_term.delete('stat_token_total_by_day:{}'.format(item_date)) - r_serv_term.srem('stat_token_history', item_date) - -def get_all_token_stat_history(): - return r_serv_term.smembers('stat_token_history') - -def get_tracked_term_last_updated_by_type(term_type): - epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type)) - if not epoch_update: - epoch_update = 0 - return float(epoch_update) - -def parse_get_tracker_term_item(dict_input, user_id): - term_uuid = dict_input.get('uuid', None) - res = check_term_uuid_valid_access(term_uuid, user_id) - if res: - return res - - date_from = dict_input.get('date_from', None) - date_to = dict_input.get('date_to', None) - - if date_from is None: - date_from = get_tracked_term_first_seen(term_uuid) - if date_from: - date_from = date_from[0] - - if date_to is None: - date_to = date_from - - if date_from > date_to: - date_from = date_to - - all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to) - all_item_id = Items.get_item_list_desc(all_item_id) - - res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id} - return res_dict, 200 - -def get_tracked_term_first_seen(term_uuid): - return Tracker.get_tracker_first_seen(term_uuid) - -def get_tracked_term_last_seen(term_uuid): - return Tracker.get_tracker_last_seen(term_uuid) - -def get_term_metedata(term_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False): - dict_uuid = {} - dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') - dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') - dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date') - dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid) - dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid) - if user_id: - dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') - if level: - dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') - if mails: - dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid) - if tags: - dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid) - if sparkline: - dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid) - if description: - dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description') - dict_uuid['uuid'] = term_uuid - return dict_uuid - -def get_tracked_term_sparkline(tracker_uuid, num_day=6): - date_range_sparkline = Date.get_date_range(num_day) - sparklines_value = [] - for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - sparklines_value.append(int(nb_seen_this_day)) - return sparklines_value - -def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None): - if date_from and date_to: - date_range = Date.substract_date(date_from, date_to) - else: - date_range = Date.get_date_range(num_day) - list_tracker_stats = [] - for tracker_uuid in list_tracker_uuid: - dict_tracker_data = [] - tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked') - for date_day in date_range: - nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)}) - list_tracker_stats.append({"name": tracker, "Data": dict_tracker_data}) - return list_tracker_stats - -def get_list_trackeed_term_tags(term_uuid): - res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)) - if res: - return list(res) - else: - return [] - -def get_list_trackeed_term_mails(term_uuid): - res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)) - if res: - return list(res) - else: - return [] - -def get_user_tracked_term_uuid(user_id, filter_type=None): - if filter_type: - return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id, filter_type))) - else: - return list(r_serv_term.smembers('user:tracker:{}'.format(user_id))) - -def get_global_tracked_term_uuid(filter_type=None): - if filter_type: - return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type))) - else: - return list(r_serv_term.smembers('global:tracker')) - -def get_all_user_tracked_terms(user_id, filter_type=None): - all_user_term = [] - all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type) - - for term_uuid in all_user_term_uuid: - all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True)) - return all_user_term - -def get_all_global_tracked_terms(filter_type=None): - all_user_term = [] - all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type) - - for term_uuid in all_user_term_uuid: - all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True)) - return all_user_term diff --git a/bin/packages/lib_words.py b/bin/packages/lib_words.py index e44a922c..7e38c472 100644 --- a/bin/packages/lib_words.py +++ b/bin/packages/lib_words.py @@ -3,8 +3,6 @@ import os import string -from pubsublogger import publisher - import calendar from datetime import date from dateutil.rrule import rrule, DAILY diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index a62cf19c..1cd9b784 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -61,15 +61,17 @@ class Retro_Hunt(AbstractModule): self.progress = 0 # First launch # restart - rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True) + retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF - timeout = Tracker.get_retro_hunt_task_timeout(task_uuid) + rule = retro_hunt.get_rule(r_compile=True) + + timeout = retro_hunt.get_timeout() self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') - sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True) + sources = retro_hunt.get_sources(r_sort=True) - self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid) - self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid) - self.tags = Tracker.get_retro_hunt_task_tags(task_uuid) + self.date_from = retro_hunt.get_date_from() + self.date_to = retro_hunt.get_date_to() + self.tags = retro_hunt.get_tags() curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid) self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources) self.update_progress(sources, curr_date) @@ -106,11 +108,10 @@ class Retro_Hunt(AbstractModule): # PAUSE self.update_progress(sources, curr_date) - if Tracker.check_retro_hunt_pause(task_uuid): + if retro_hunt.to_pause(): Tracker.set_retro_hunt_last_analyzed(task_uuid, id) # self.update_progress(sources, curr_date, save_db=True) - Tracker.pause_retro_hunt_task(task_uuid) - Tracker.clear_retro_hunt_task_cache(task_uuid) + retro_hunt.pause() return None self.nb_src_done += 1 @@ -120,9 +121,7 @@ class Retro_Hunt(AbstractModule): self.update_progress(sources, curr_date) - Tracker.set_retro_hunt_task_state(task_uuid, 'completed') - Tracker.set_retro_hunt_nb_match(task_uuid) - Tracker.clear_retro_hunt_task_cache(task_uuid) + retro_hunt.complete() print(f'Retro Hunt {task_uuid} completed') self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') @@ -130,10 +129,11 @@ class Retro_Hunt(AbstractModule): # # TODO: stop def update_progress(self, sources, curr_date, save_db=False): - progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to, - sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done) + retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF + progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to, + sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done) if self.progress != progress: - Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress) + retro_hunt.set_progress(progress) self.progress = progress # if save_db: # Tracker.set_retro_hunt_task_progress(task_uuid, progress) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index c8d54212..56de7696 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -17,9 +17,8 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item +from lib.objects import ail_objects from lib.ConfigLoader import ConfigLoader -from packages import Term from lib import Tracker from exporter.MailExporter import MailExporterTracker @@ -39,7 +38,7 @@ class Tracker_Regex(AbstractModule): self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time") # refresh Tracked Regex - self.dict_regex_tracked = Term.get_regex_tracked_words_dict() + self.tracked_regexs = Tracker.get_tracked_regexs() self.last_refresh = time.time() # Exporter @@ -48,56 +47,65 @@ class Tracker_Regex(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, item_id, content=None): + def compute(self, obj_id, obj_type='item', subtype=''): # refresh Tracked regex if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): - self.dict_regex_tracked = Term.get_regex_tracked_words_dict() + self.tracked_regexs = Tracker.get_tracked_regexs() self.last_refresh = time.time() self.redis_logger.debug('Tracked regex refreshed') print('Tracked regex refreshed') - item = Item(item_id) - item_id = item.get_id() - if not content: - content = item.get_content() + obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj_id = obj.get_id() + obj_type = obj.get_type() - for regex in self.dict_regex_tracked: - matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content) + # Object Filter + if obj_type not in self.tracked_regexs: + return None + + content = obj.get_content(r_str=True) + + for dict_regex in self.tracked_regexs[obj_type]: + matched = self.regex_findall(dict_regex['regex'], obj_id, content) if matched: - self.new_tracker_found(regex, 'regex', item) + self.new_tracker_found(dict_regex['tracked'], 'regex', obj) - def new_tracker_found(self, tracker_name, tracker_type, item): - uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) - - item_id = item.get_id() - # date = item.get_date() - item_source = item.get_source() - - for tracker_uuid in uuid_list: + def new_tracker_found(self, tracker_name, tracker_type, obj): + obj_id = obj.get_id() + for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name): tracker = Tracker.Tracker(tracker_uuid) - # Source Filtering - tracker_sources = tracker.get_sources() - if tracker_sources and item_source not in tracker_sources: + # Filter Object + filters = tracker.get_filters() + if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked regex found: {tracker_name} in {item_id}') - self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}') - # TODO - Tracker.add_tracked_item(tracker_uuid, item_id) + print(f'new tracked regex found: {tracker_name} in {obj_id}') + self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}') + + if obj.get_type() == 'item': + date = obj.get_date() + else: + date = None + + tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date) for tag in tracker.get_tags(): - msg = f'{tag};{item_id}' - self.add_message_to_queue(msg, 'Tags') + if obj.get_type() == 'item': + msg = f'{tag};{obj_id}' + self.add_message_to_queue(msg, 'Tags') + else: + obj.add_tag(tag) if tracker.mail_export(): # TODO add matches + custom subjects - self.exporters['mail'].export(tracker, item) + self.exporters['mail'].export(tracker, obj) if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, item) + self.exporters['webhook'].export(tracker, obj) if __name__ == "__main__": module = Tracker_Regex() module.run() + # module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz') diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 86c6a87b..1fabddce 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -21,8 +21,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item -from packages import Term +from lib.objects import ail_objects from lib import Tracker from exporter.MailExporter import MailExporterTracker @@ -54,9 +53,9 @@ class Tracker_Term(AbstractModule): self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time") # loads tracked words - self.list_tracked_words = Term.get_tracked_words_list() + self.tracked_words = Tracker.get_tracked_words() self.last_refresh_word = time.time() - self.set_tracked_words_list = Term.get_set_tracked_words_list() + self.tracked_sets = Tracker.get_tracked_sets() self.last_refresh_set = time.time() # Exporter @@ -65,93 +64,94 @@ class Tracker_Term(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, item_id, item_content=None): + def compute(self, obj_id, obj_type='item', subtype=''): # refresh Tracked term - if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'): - self.list_tracked_words = Term.get_tracked_words_list() + if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'): + self.tracked_words = Tracker.get_tracked_words() self.last_refresh_word = time.time() self.redis_logger.debug('Tracked word refreshed') print('Tracked word refreshed') - if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'): - self.set_tracked_words_list = Term.get_set_tracked_words_list() + if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'): + self.tracked_sets = Tracker.get_tracked_sets() self.last_refresh_set = time.time() self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') - # Cast message as Item - item = Item(item_id) - if not item_content: - item_content = item.get_content() + obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj_type = obj.get_type() + + # Object Filter + if obj_type not in self.tracked_words and obj_type not in self.tracked_sets: + return None + + content = obj.get_content(r_str=True) signal.alarm(self.max_execution_time) dict_words_freq = None try: - dict_words_freq = Term.get_text_word_frequency(item_content) + dict_words_freq = Tracker.get_text_word_frequency(content) except TimeoutException: - self.redis_logger.warning(f"{item.get_id()} processing timeout") + self.redis_logger.warning(f"{obj.get_id()} processing timeout") else: signal.alarm(0) if dict_words_freq: - # create token statistics - # for word in dict_words_freq: - # Term.create_token_statistics(item_date, word, dict_words_freq[word]) # check solo words - # ###### # TODO: check if source needed ####### - for word in self.list_tracked_words: + for word in self.tracked_words[obj_type]: if word in dict_words_freq: - self.new_term_found(word, 'word', item) + self.new_tracker_found(word, 'word', obj) # check words set - for elem in self.set_tracked_words_list: - list_words = elem[0] - nb_words_threshold = elem[1] - word_set = elem[2] + for tracked_set in self.tracked_sets[obj_type]: nb_uniq_word = 0 - - for word in list_words: + for word in tracked_set['words']: if word in dict_words_freq: nb_uniq_word += 1 - if nb_uniq_word >= nb_words_threshold: - self.new_term_found(word_set, 'set', item) + if nb_uniq_word >= tracked_set['nb']: + self.new_tracker_found(tracked_set['tracked'], 'set', obj) - def new_term_found(self, tracker_name, tracker_type, item): - uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) + def new_tracker_found(self, tracker_name, tracker_type, obj): # TODO FILTER + obj_id = obj.get_id() - item_id = item.get_id() - item_source = item.get_source() - - for tracker_uuid in uuid_list: + for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name): tracker = Tracker.Tracker(tracker_uuid) - # Source Filtering - tracker_sources = tracker.get_sources() - if tracker_sources and item_source not in tracker_sources: + # Filter Object + filters = tracker.get_filters() + if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked term found: {tracker_name} in {item_id}') - self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}') - # TODO - Tracker.add_tracked_item(tracker_uuid, item_id) + print(f'new tracked term found: {tracker_name} in {obj_id}') + self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}') + + if obj.get_type() == 'item': + date = obj.get_date() + else: + date = None + tracker.add(obj.get_type(), obj.get_subtype(), obj_id, date=date) # Tags for tag in tracker.get_tags(): - msg = f'{tag};{item_id}' - self.add_message_to_queue(msg, 'Tags') + if obj.get_type() == 'item': + msg = f'{tag};{obj_id}' + self.add_message_to_queue(msg, 'Tags') + else: + obj.add_tag(tag) # Mail if tracker.mail_export(): # TODO add matches + custom subjects - self.exporters['mail'].export(tracker, item) + self.exporters['mail'].export(tracker, obj) # Webhook if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, item) + self.exporters['webhook'].export(tracker, obj) if __name__ == '__main__': module = Tracker_Term() module.run() + # module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz') diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index fa7f613b..0a573512 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item +from lib.objects import ail_objects from lib import Tracker from exporter.MailExporter import MailExporterTracker @@ -36,8 +36,8 @@ class Tracker_Typo_Squatting(AbstractModule): self.pending_seconds = 5 # Refresh typo squatting - self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() - self.last_refresh_typosquat = time.time() + self.tracked_typosquattings = Tracker.get_tracked_typosquatting() + self.last_refresh_typosquatting = time.time() # Exporter self.exporters = {'mail': MailExporterTracker(), @@ -45,50 +45,62 @@ class Tracker_Typo_Squatting(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, message): + def compute(self, message, obj_type='item', subtype=''): # refresh Tracked typo - if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'): - self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() - self.last_refresh_typosquat = time.time() + if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'): + self.tracked_typosquattings = Tracker.get_tracked_typosquatting() + self.last_refresh_typosquatting = time.time() self.redis_logger.debug('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed') - host, item_id = message.split() + host, obj_id = message.split() + obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj_type = obj.get_type() - # Cast message as Item - for tracker in self.typosquat_tracked_words_list: - if host in self.typosquat_tracked_words_list[tracker]: - item = Item(item_id) - self.new_tracker_found(tracker, 'typosquatting', item) + # Object Filter + if obj_type not in self.tracked_typosquattings: + return None - def new_tracker_found(self, tracker, tracker_type, item): - item_id = item.get_id() - item_source = item.get_source() - print(f'new tracked typosquatting found: {tracker} in {item_id}') - self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}') + for typo in self.tracked_typosquattings[obj_type]: + if host in typo['domains']: + self.new_tracker_found(typo['tracked'], 'typosquatting', obj) - for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type): + def new_tracker_found(self, tracked, tracker_type, obj): + obj_id = obj.get_id() + for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracked): tracker = Tracker.Tracker(tracker_uuid) - # Source Filtering - tracker_sources = tracker.get_sources() - if tracker_sources and item_source not in tracker_sources: + # Filter Object + filters = tracker.get_filters() + if ail_objects.is_filtered(obj, filters): continue - Tracker.add_tracked_item(tracker_uuid, item_id) + print(f'new tracked typosquatting found: {tracked} in {obj_id}') + self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}') + if obj.get_type() == 'item': + date = obj.get_date() + else: + date = None + + tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date) + + # Tags for tag in tracker.get_tags(): - msg = f'{tag};{item_id}' - self.add_message_to_queue(msg, 'Tags') + if obj.get_type() == 'item': + msg = f'{tag};{obj_id}' + self.add_message_to_queue(msg, 'Tags') + else: + obj.add_tag(tag) if tracker.mail_export(): - self.exporters['mail'].export(tracker, item) + self.exporters['mail'].export(tracker, obj) if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, item) + self.exporters['webhook'].export(tracker, obj) if __name__ == '__main__': module = Tracker_Typo_Squatting() module.run() - #module.compute('g00gle.com tests/2020/01/01/test.gz') + # module.compute('g00gle.com tests/2020/01/01/test.gz') diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 312006f8..98d7d4cf 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item +from lib.objects import ail_objects from lib import Tracker from exporter.MailExporter import MailExporterTracker @@ -35,10 +35,10 @@ class Tracker_Yara(AbstractModule): self.pending_seconds = 5 # Load Yara rules - self.rules = Tracker.reload_yara_rules() + self.rules = Tracker.get_tracked_yara_rules() self.last_refresh = time.time() - self.item = None + self.obj = None # Exporter self.exporters = {'mail': MailExporterTracker(), @@ -46,58 +46,67 @@ class Tracker_Yara(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, item_id, item_content=None): + def compute(self, obj_id, obj_type='item', subtype=''): # refresh YARA list if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): - self.rules = Tracker.reload_yara_rules() + self.rules = Tracker.get_tracked_yara_rules() self.last_refresh = time.time() self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') - self.item = Item(item_id) - if not item_content: - item_content = self.item.get_content() + self.obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj_type = self.obj.get_type() + + # Object Filter + if obj_type not in self.rules: + return None + + content = self.obj.get_content(r_str=True) try: - yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, - which_callbacks=yara.CALLBACK_MATCHES, timeout=60) + yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, + which_callbacks=yara.CALLBACK_MATCHES, timeout=60) if yara_match: - self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}') - print(f'{self.item.get_id()}: {yara_match}') + self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}') + print(f'{self.obj.get_id()}: {yara_match}') except yara.TimeoutError: - print(f'{self.item.get_id()}: yara scanning timed out') - self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out') + print(f'{self.obj.get_id()}: yara scanning timed out') + self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') def yara_rules_match(self, data): - tracker_uuid = data['namespace'] - item_id = self.item.get_id() + tracker_name = data['namespace'] + obj_id = self.obj.get_id() + for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name): + tracker = Tracker.Tracker(tracker_uuid) - item = Item(item_id) - item_source = self.item.get_source() + # Filter Object + filters = tracker.get_filters() + if ail_objects.is_filtered(self.obj, filters): + continue - tracker = Tracker.Tracker(tracker_uuid) + if self.obj.get_type() == 'item': + date = self.obj.get_date() + else: + date = None - # Source Filtering - tracker_sources = tracker.get_sources() - if tracker_sources and item_source not in tracker_sources: - print(f'Source Filtering: {data["rule"]}') - return yara.CALLBACK_CONTINUE + tracker.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id, date=date) - Tracker.add_tracked_item(tracker_uuid, item_id) # TODO + # Tags + for tag in tracker.get_tags(): + if self.obj.get_type() == 'item': + msg = f'{tag};{obj_id}' + self.add_message_to_queue(msg, 'Tags') + else: + self.obj.add_tag(tag) - # Tags - for tag in tracker.get_tags(): - msg = f'{tag};{item_id}' - self.add_message_to_queue(msg, 'Tags') + # Mails + if tracker.mail_export(): + # TODO add matches + custom subjects + self.exporters['mail'].export(tracker, self.obj) - # Mails - if tracker.mail_export(): - # TODO add matches + custom subjects - self.exporters['mail'].export(tracker, item) - - # Webhook - if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, item) + # Webhook + if tracker.webhook_export(): + self.exporters['webhook'].export(tracker, self.obj) return yara.CALLBACK_CONTINUE @@ -105,3 +114,4 @@ class Tracker_Yara(AbstractModule): if __name__ == '__main__': module = Tracker_Yara() module.run() + # module.compute('archive/gist.github.com/2023/04/13/chipzoller_d8d6d2d737d02ad4fe9d30a897170761.gz') diff --git a/update/v2.2/Update.py b/update/v2.2/Update.py index 8d7ec233..2bfef8e9 100755 --- a/update/v2.2/Update.py +++ b/update/v2.2/Update.py @@ -39,74 +39,75 @@ if __name__ == '__main__': r_serv_term_stats.flushdb() - # convert all regex: - all_regex = r_serv_termfreq.smembers('TrackedRegexSet') - for regex in all_regex: - tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex))) - mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex))) - - new_term = regex[1:-1] - res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, - 'admin@admin.test') - if res[1] == 200: - term_uuid = res[0]['uuid'] - list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) - for paste_item in list_items: - item_id = get_item_id(paste_item) - item_date = get_item_date(item_id) - Term.add_tracked_item(term_uuid, item_id, item_date) - - # Invalid Tracker => remove it - else: - print('Invalid Regex Removed: {}'.format(regex)) - print(res[0]) - # allow reprocess - r_serv_termfreq.srem('TrackedRegexSet', regex) - - all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') - for token in all_tokens: - tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token))) - mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token))) - - res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') - if res[1] == 200: - term_uuid = res[0]['uuid'] - list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) - for paste_item in list_items: - item_id = get_item_id(paste_item) - item_date = get_item_date(item_id) - Term.add_tracked_item(term_uuid, item_id, item_date) - # Invalid Tracker => remove it - else: - print('Invalid Token Removed: {}'.format(token)) - print(res[0]) - # allow reprocess - r_serv_termfreq.srem('TrackedSetTermSet', token) - - all_set = r_serv_termfreq.smembers('TrackedSetSet') - for curr_set in all_set: - tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set))) - mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set))) - - to_remove = ',{}'.format(curr_set.split(',')[-1]) - new_set = rreplace(curr_set, to_remove, '', 1) - new_set = new_set[2:] - new_set = new_set.replace(',', '') - - res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') - if res[1] == 200: - term_uuid = res[0]['uuid'] - list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) - for paste_item in list_items: - item_id = get_item_id(paste_item) - item_date = get_item_date(item_id) - Term.add_tracked_item(term_uuid, item_id, item_date) - # Invalid Tracker => remove it - else: - print('Invalid Set Removed: {}'.format(curr_set)) - print(res[0]) - # allow reprocess - r_serv_termfreq.srem('TrackedSetSet', curr_set) + # Disabled. Checkout the v2.2 branch if you need it + # # convert all regex: + # all_regex = r_serv_termfreq.smembers('TrackedRegexSet') + # for regex in all_regex: + # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex))) + # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex))) + # + # new_term = regex[1:-1] + # res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, + # 'admin@admin.test') + # if res[1] == 200: + # term_uuid = res[0]['uuid'] + # list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) + # for paste_item in list_items: + # item_id = get_item_id(paste_item) + # item_date = get_item_date(item_id) + # Term.add_tracked_item(term_uuid, item_id, item_date) + # + # # Invalid Tracker => remove it + # else: + # print('Invalid Regex Removed: {}'.format(regex)) + # print(res[0]) + # # allow reprocess + # r_serv_termfreq.srem('TrackedRegexSet', regex) + # + # all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') + # for token in all_tokens: + # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token))) + # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token))) + # + # res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + # if res[1] == 200: + # term_uuid = res[0]['uuid'] + # list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) + # for paste_item in list_items: + # item_id = get_item_id(paste_item) + # item_date = get_item_date(item_id) + # Term.add_tracked_item(term_uuid, item_id, item_date) + # # Invalid Tracker => remove it + # else: + # print('Invalid Token Removed: {}'.format(token)) + # print(res[0]) + # # allow reprocess + # r_serv_termfreq.srem('TrackedSetTermSet', token) + # + # all_set = r_serv_termfreq.smembers('TrackedSetSet') + # for curr_set in all_set: + # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set))) + # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set))) + # + # to_remove = ',{}'.format(curr_set.split(',')[-1]) + # new_set = rreplace(curr_set, to_remove, '', 1) + # new_set = new_set[2:] + # new_set = new_set.replace(',', '') + # + # res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + # if res[1] == 200: + # term_uuid = res[0]['uuid'] + # list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) + # for paste_item in list_items: + # item_id = get_item_id(paste_item) + # item_date = get_item_date(item_id) + # Term.add_tracked_item(term_uuid, item_id, item_date) + # # Invalid Tracker => remove it + # else: + # print('Invalid Set Removed: {}'.format(curr_set)) + # print(res[0]) + # # allow reprocess + # r_serv_termfreq.srem('TrackedSetSet', curr_set) r_serv_termfreq.flushdb() diff --git a/update/v3.7/Update.py b/update/v3.7/Update.py index f1f988e8..8b238f9c 100755 --- a/update/v3.7/Update.py +++ b/update/v3.7/Update.py @@ -24,7 +24,7 @@ class Updater(AIL_Updater): print('Fixing Tracker_uuid list ...') Tracker.fix_all_tracker_uuid_list() nb = 0 - for tracker_uuid in Tracker.get_all_tracker_uuid(): + for tracker_uuid in Tracker.get_trackers(): self.r_serv.sadd('trackers_update_v3.7', tracker_uuid) nb += 1 diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index 5ac5feb3..e15fae35 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... -''' +""" import os import sys import json -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response +from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -22,8 +22,10 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib import ail_core from lib import item_basic from lib import Tracker +from lib import Tag bootstrap_label = Flask_config.bootstrap_label @@ -34,7 +36,6 @@ hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ # ============ VARIABLES ============ - # ============ FUNCTIONS ============ def api_validator(api_response): if api_response: @@ -45,11 +46,203 @@ def create_json_response(data, status_code): # ============= ROUTES ============== +################## +# TRACKERS # +################## + +@hunters.route('/trackers', methods=['GET']) +@login_required +@login_read_only +def trackers_dashboard(): + user_id = current_user.get_id() # TODO + trackers = Tracker.get_trackers_dashboard() + stats = Tracker.get_trackers_stats(user_id) + return render_template("trackers_dashboard.html", trackers=trackers, stats=stats, bootstrap_label=bootstrap_label) + +@hunters.route("/trackers/all") +@login_required +@login_read_only +def tracked_menu(): + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id) + global_trackers = Tracker.get_global_trackers_meta() + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label) + +@hunters.route("/trackers/word") +@login_required +@login_read_only +def tracked_menu_word(): + tracker_type = 'word' + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type='word') + global_trackers = Tracker.get_global_trackers_meta(tracker_type='word') + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) + +@hunters.route("/trackers/set") +@login_required +@login_read_only +def tracked_menu_set(): + tracker_type = 'set' + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) + +@hunters.route("/trackers/regex") +@login_required +@login_read_only +def tracked_menu_regex(): + tracker_type = 'regex' + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) + +@hunters.route("/trackers/yara") +@login_required +@login_read_only +def tracked_menu_yara(): + tracker_type = 'yara' + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) + +@hunters.route("/trackers/typosquatting") +@login_required +@login_read_only +def tracked_menu_typosquatting(): + tracker_type = 'typosquatting' + user_id = current_user.get_id() + user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, + bootstrap_label=bootstrap_label, tracker_type=tracker_type) + +@hunters.route("/tracker/add", methods=['GET', 'POST']) +@login_required +@login_analyst +def add_tracked_menu(): + if request.method == 'POST': + to_track = request.form.get("tracker") + tracker_uuid = request.form.get("tracker_uuid") + tracker_type = request.form.get("tracker_type") + nb_words = request.form.get("nb_word", 1) + description = request.form.get("description", '') + webhook = request.form.get("webhook", '') + level = request.form.get("level", 0) + mails = request.form.get("mails", []) + + # TAGS + tags = request.form.get("tags", []) + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except: + galaxies_tags = [] + # custom tags + if tags: + tags = tags.split() + else: + tags = [] + tags = tags + taxonomies_tags + galaxies_tags + + # YARA # + if tracker_type == 'yara': + yara_default_rule = request.form.get("yara_default_rule") + yara_custom_rule = request.form.get("yara_custom_rule") + if yara_custom_rule: + to_track = yara_custom_rule + tracker_type = 'yara_custom' + else: + to_track = yara_default_rule + tracker_type = 'yara_default' + + if level == 'on': + level = 1 + else: + level = 0 + if mails: + mails = mails.split() + else: + tags = [] + + # FILTERS + filters = {} + for obj_type in Tracker.get_objects_tracked(): + new_filter = request.form.get(f'{obj_type}_obj') + if new_filter == 'on': + filters[obj_type] = {} + # Mimetypes + mimetypes = request.form.get(f'mimetypes_{obj_type}', []) + if mimetypes: + mimetypes = json.loads(mimetypes) + filters[obj_type]['mimetypes'] = mimetypes + # Sources + sources = request.form.get(f'sources_{obj_type}', []) + if sources: + sources = json.loads(sources) + filters[obj_type]['sources'] = sources + # Subtypes + for obj_subtype in ail_core.get_object_all_subtypes(obj_type): + subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}') + if subtype == 'on': + if 'subtypes' not in filters[obj_type]: + filters[obj_type]['subtypes'] = [] + filters[obj_type]['subtypes'].append(obj_subtype) + + input_dict = {"tracked": to_track, "type": tracker_type, + "tags": tags, "mails": mails, "filters": filters, + "level": level, "description": description, "webhook": webhook} + if tracker_type == 'set': + try: + input_dict['nb_words'] = int(nb_words) + except TypeError: + input_dict['nb_words'] = 1 + + user_id = current_user.get_id() + res = Tracker.api_add_tracker(input_dict, user_id) + if res[1] == 200: + return redirect(url_for('hunters.trackers_dashboard')) + else: + return create_json_response(res[0], res[1]) + else: + return render_template("tracker_add.html", + all_sources=item_basic.get_all_items_sources(r_list=True), + tags_selector_data=Tag.get_tags_selector_data(), + all_yara_files=Tracker.get_all_default_yara_files()) + +@hunters.route('/tracker/delete', methods=['GET']) +@login_required +@login_analyst +def tracker_delete(): + user_id = current_user.get_id() + tracker_uuid = request.args.get('uuid') + res = Tracker.api_delete_tracker({'uuid': tracker_uuid}, user_id) + if res[1] != 200: + return create_json_response(res[0], res[1]) + else: + return redirect(url_for('hunter.tracked_menu')) + + +#################### +# RETRO HUNT # +#################### + @hunters.route('/retro_hunt/tasks', methods=['GET']) @login_required @login_read_only def retro_hunt_all_tasks(): - retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata() + retro_hunts = Tracker.get_retro_hunt_tasks_metas() return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label) @hunters.route('/retro_hunt/task/show', methods=['GET']) @@ -69,8 +262,8 @@ def retro_hunt_show_task(): if res: return create_json_response(res[0], res[1]) - dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True, - sources=True, tags=True, description=True) + retro_hunt = Tracker.RetroHunt(task_uuid) + dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'}) rule_content = Tracker.get_yara_rule_content(dict_task['rule']) if date_from: @@ -177,7 +370,7 @@ def retro_hunt_delete_task(): #### JSON #### -@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET']) +@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET']) @login_required @login_read_only def get_json_retro_hunt_nb_items_by_date(): diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 921581e1..e999f280 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -17,10 +17,11 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib.objects import ail_objects from lib import item_basic from lib import Tracker from lib import Tag -from packages import Term +from packages import Date # ============ VARIABLES ============ @@ -34,142 +35,11 @@ hunter = Blueprint('hunter', __name__, template_folder='templates') # ============ FUNCTIONS ============ +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + # ============ ROUTES ============ -@hunter.route("/trackers") -@login_required -@login_read_only -def tracked_menu(): - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id) - global_trackers = Tracker.get_global_trackers_metadata() - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label) - -@hunter.route("/trackers/word") -@login_required -@login_read_only -def tracked_menu_word(): - tracker_type = 'word' - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word') - global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word') - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) - -@hunter.route("/trackers/set") -@login_required -@login_read_only -def tracked_menu_set(): - tracker_type = 'set' - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) - global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) - -@hunter.route("/trackers/regex") -@login_required -@login_read_only -def tracked_menu_regex(): - tracker_type = 'regex' - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) - global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) - -@hunter.route("/trackers/yara") -@login_required -@login_read_only -def tracked_menu_yara(): - tracker_type = 'yara' - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) - global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) - -@hunter.route("/trackers/typosquatting") -@login_required -@login_read_only -def tracked_menu_typosquatting(): - tracker_type = 'typosquatting' - user_id = current_user.get_id() - user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) - global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) - return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, - bootstrap_label=bootstrap_label, tracker_type=tracker_type) - -@hunter.route("/tracker/add", methods=['GET', 'POST']) -@login_required -@login_analyst -def add_tracked_menu(): - if request.method == 'POST': - tracker = request.form.get("tracker") - tracker_uuid = request.form.get("tracker_uuid") - tracker_type = request.form.get("tracker_type") - nb_words = request.form.get("nb_word", 1) - description = request.form.get("description", '') - webhook = request.form.get("webhook", '') - level = request.form.get("level", 0) - mails = request.form.get("mails", []) - sources = request.form.get("sources", []) - - tags = request.form.get("tags", []) - taxonomies_tags = request.form.get('taxonomies_tags') - if taxonomies_tags: - try: - taxonomies_tags = json.loads(taxonomies_tags) - except Exception: - taxonomies_tags = [] - else: - taxonomies_tags = [] - galaxies_tags = request.form.get('galaxies_tags') - if galaxies_tags: - try: - galaxies_tags = json.loads(galaxies_tags) - except Exception: - galaxies_tags = [] - - - # YARA # - if tracker_type == 'yara': - yara_default_rule = request.form.get("yara_default_rule") - yara_custom_rule = request.form.get("yara_custom_rule") - if yara_custom_rule: - tracker = yara_custom_rule - tracker_type='yara_custom' - else: - tracker = yara_default_rule - tracker_type='yara_default' - - if level == 'on': - level = 1 - if mails: - mails = mails.split() - if tags: - tags = tags.split() - if sources: - sources = json.loads(sources) - - input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words, - "tags": tags, "mails": mails, "sources": sources, - "level": level, "description": description, "webhook": webhook} - user_id = current_user.get_id() - # edit tracker - if tracker_uuid: - input_dict['uuid'] = tracker_uuid - res = Tracker.api_add_tracker(input_dict, user_id) - if res[1] == 200: - if 'uuid' in res[0]: - return redirect(url_for('hunter.show_tracker', uuid=res[0]['uuid'])) - else: - return redirect(url_for('hunter.tracked_menu')) - else: - ## TODO: use modal - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - else: - return render_template("edit_tracker.html", - all_sources=item_basic.get_all_items_sources(r_list=True), - tags_selector_data=Tag.get_tags_selector_data(), - all_yara_files=Tracker.get_all_default_yara_files()) - @hunter.route("/tracker/edit", methods=['GET', 'POST']) @login_required @login_analyst @@ -181,7 +51,8 @@ def edit_tracked_menu(): if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) + tracker = Tracker.Tracker(tracker_uuid) + dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'sources', 'tags', 'user', 'webhook'}) dict_tracker['tags'] = ' '.join(dict_tracker['tags']) dict_tracker['mails'] = ' '.join(dict_tracker['mails']) @@ -205,7 +76,7 @@ def edit_tracked_menu(): # word # set of word + nb words # regex - # yara custum + # yara custom # yara default ???? => allow edit ? #### EDIT SHow Trackers ?????????????????????????????????????????????????? @@ -228,34 +99,36 @@ def show_tracker(): if date_to: date_to = date_to.replace('-', '') - tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True) + tracker = Tracker.Tracker(tracker_uuid) + meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags', + 'user', 'webhook'}) - if tracker_metadata['type'] == 'yara': - yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) + if meta['type'] == 'yara': + yara_rule_content = Tracker.get_yara_rule_content(meta['tracked']) else: yara_rule_content = None - if tracker_metadata['type'] == 'typosquatting': - typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid)) - typo_squatting.sort() + if meta['type'] == 'typosquatting': + typo_squatting = Tracker.get_tracked_typosquatting_domains(meta['tracked']) + sorted(typo_squatting) else: - typo_squatting = None + typo_squatting = set() if date_from: - res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) - if res[1] != 200: - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - tracker_metadata['items'] = res[0]['items'] - tracker_metadata['date_from'] = res[0]['date_from'] - tracker_metadata['date_to'] = res[0]['date_to'] + date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen']) + objs = tracker.get_objs_by_daterange(date_from, date_to) + meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True) else: - tracker_metadata['items'] = [] - tracker_metadata['date_from'] = '' - tracker_metadata['date_to'] = '' + date_from = '' + date_to = '' + meta['objs'] = [] - tracker_metadata['sources'] = sorted(tracker_metadata['sources']) + meta['date_from'] = date_from + meta['date_to'] = date_to + print(meta['filters']) + meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', [])) - return render_template("showTracker.html", tracker_metadata=tracker_metadata, + return render_template("showTracker.html", tracker_metadata=meta, yara_rule_content=yara_rule_content, typo_squatting=typo_squatting, bootstrap_label=bootstrap_label) @@ -309,21 +182,16 @@ def update_tracker_mails(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) -@hunter.route("/tracker/delete", methods=['GET']) -@login_required -@login_analyst -def delete_tracker(): - user_id = current_user.get_id() - term_uuid = request.args.get('uuid') - res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id) - if res[1] !=200: - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - return redirect(url_for('hunter.tracked_menu')) - @hunter.route("/tracker/get_json_tracker_stats", methods=['GET']) @login_required @login_read_only def get_json_tracker_stats(): + user_id = current_user.get_id() + tracker_uuid = request.args.get('uuid') + res = Tracker.api_check_tracker_acl(tracker_uuid, user_id) + if res: + return create_json_response(res[0], res[1]) + date_from = request.args.get('date_from') date_to = request.args.get('date_to') @@ -331,13 +199,10 @@ def get_json_tracker_stats(): date_from = date_from.replace('-', '') if date_to: date_to = date_to.replace('-', '') - - tracker_uuid = request.args.get('uuid') - if date_from and date_to: - res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to) + res = Tracker.get_trackers_graph_by_day([tracker_uuid], date_from=date_from, date_to=date_to) else: - res = Term.get_list_tracked_term_stats_by_day([tracker_uuid]) + res = Tracker.get_trackers_graph_by_day([tracker_uuid]) return jsonify(res) @hunter.route("/tracker/yara/default_rule/content", methods=['GET']) @@ -348,5 +213,6 @@ def get_default_yara_rule_content(): res = Tracker.api_get_default_rule_content(default_yara_rule) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + # ========= REGISTRATION ========= app.register_blueprint(hunter, url_prefix=baseUrl) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html deleted file mode 100644 index 39892b37..00000000 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ /dev/null @@ -1,260 +0,0 @@ - - - - - AIL-Framework - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'hunter/menu_sidebar.html' %} - -
- -
-
-
{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker
-
-
- -
- {%if dict_tracker%} - - {%endif%} - -
-
-
-
-
-
- -
-
-
-
-
- -
-
-
-
-
- -
- -
-
-
-
- -
- - -
-
- Tags -
-
-
-
-
-
- -
- {% include 'tags/block_tags_selector.html' %} -
-
- -
-
-
- - -
-
-
- -
-

Tracker Type:

- - - -

Terms to track (space separated)

- -
-
- -
-
- -
-
- - -
-
- -
Default YARA rules:
- - - 

-
-										

-
-										

-
-										
Custom YARA rules:

-										

-											
-										

-									

-
-									

-									
-
-								

-
-
-
-						  

-						

-
-
-			

-
-		

-	

-
-
-
-
diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html
index ca6fdf6b..466f91f5 100644
--- a/var/www/modules/hunter/templates/showTracker.html
+++ b/var/www/modules/hunter/templates/showTracker.html
@@ -72,7 +72,7 @@
                                             
                                                 Type
                                                 Tracker
-                                                Date added
+                                                Created
                                                 Access Level
                                                 Created by
                                                 First seen
@@ -94,7 +94,7 @@
                                             {% if tracker_metadata['type'] == 'typosquatting' %}
                                                 
                                                     
                                                     

                                                         

@@ -108,7 +108,7 @@
                                                     

                                                 
                                             {% else %}
-                                                {{ tracker_metadata['tracker'] }}
+                                                {{ tracker_metadata['tracked'] }}
                                             {% endif %}
                                             {{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}
                                             
@@ -118,19 +118,15 @@
                                                     Global
                                                 {% endif %}
                                             
-                                            {{ tracker_metadata['user_id'] }}
+                                            {{ tracker_metadata['user'] }}
                                             
                                                 {% if tracker_metadata['first_seen'] %}
-                                                    {{ tracker_metadata['first_seen'][0:4] }}/
-                                                    {{ tracker_metadata['first_seen'][4:6] }}/
-                                                    {{ tracker_metadata['first_seen'][6:8] }}
+                                                    {{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }}
                                                 {% endif %}
                                             
                                             
                                                 {% if tracker_metadata['last_seen'] %}
-                                                    {{ tracker_metadata['last_seen'][0:4] }}/
-                                                    {{ tracker_metadata['last_seen'][4:6] }}/
-                                                    {{ tracker_metadata['last_seen'][6:8] }}
+                                                    {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }}
                                                 {% endif %}
                                             
                                             {% if tracker_metadata['webhook'] %}
@@ -159,14 +155,22 @@
                                     

                                 

                             
-                            
Sources:

-                            {% if tracker_metadata['sources'] %}
-                                {% for sources in tracker_metadata['sources'] %}
-                                    {{ sources }}

-                                {% endfor %}
+
+                            
Filters:

+                            {% if tracker_metadata['filters'] %}
+                                
{{ tracker_metadata['filters'] }}

                             {% else %}
-                                All Souces

+                                No Filters

                             {% endif %}
+
+{#                            
Sources:
#}
+{#                            {% if tracker_metadata['sources'] %}#}
+{#                                {% for sources in tracker_metadata['sources'] %}#}
+{#                                    {{ sources }}
#}
+{#                                {% endfor %}#}
+{#                            {% else %}#}
+{#                                All Sources
#}
+{#                            {% endif %}#}
                         
                     
 
@@ -232,7 +236,7 @@
                     
 
                     

-                        
                             
                         
@@ -262,8 +266,9 @@
                                         class="far fa-calendar-alt" aria-hidden="true">

                                 
+                                       {% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'][0:4] }}-{{ tracker_metadata['date_from'][4:6] }}-{{ tracker_metadata['date_from'][6:8] }}"
+                                       {% elif tracker_metadata['first_seen'] %}value="{{ tracker_metadata['first_seen'][0:4] }}-{{ tracker_metadata['first_seen'][4:6] }}-{{ tracker_metadata['first_seen'][6:8] }}"
+                                       {% endif %}>
                             
                         
                         

@@ -272,8 +277,9 @@
                                         class="far fa-calendar-alt" aria-hidden="true">

                                 
+                                       {% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'][0:4] }}-{{ tracker_metadata['date_to'][4:6] }}-{{ tracker_metadata['date_to'][6:8] }}"
+                                       {% elif tracker_metadata['last_seen'] %}value="{{ tracker_metadata['last_seen'][0:4] }}-{{ tracker_metadata['last_seen'][4:6] }}-{{ tracker_metadata['last_seen'][6:8] }}"
+                                       {% endif %}>
                             
                         
                     
@@ -285,40 +291,52 @@
                 
             
 
-            {% if tracker_metadata['items'] %}
+            {% if tracker_metadata['objs'] %}
                 

-                    
-                        
+                    

+                        
-                            
-                            
-                        
-                        
-                        
+                            
+			      	        
+							
+				     	    
+							
+				   	    
+					    
+					    
+						{% for object in tracker_metadata['objs'] %}
+							
+								
+								
+								
+								
+								
+							
+						{% endfor %}
+					    
+				    

                         
DateItem Id
TypeIdTags

+									{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
+										{% include 'objects/obj_svg_block.html' %}
+									{% endwith %}
+									{{ object['type']}}
+								
+									{% if object['subtype'] %}
+										{{ object['subtype']}}
+									{% endif %}
+								
+									
+										{{ object['id']}}
+									
+								
+									{% for tag in object['tags'] %}
+										{{ tag }}
+									{% endfor %}
+								
+{#									#}
+{#										#}
+{#									#}
+								

 
-                        {% for item in tracker_metadata['items'] %}
-                            
-                                
-                                    {{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }}
-                                
-                                
-                                    
-                                        
{{ item['id'] }}

-                                    
-                                    

-                                        {% for tag in item['tags'] %}
-                                            
-                                                {{ tag }}
-                                            
-                                        {% endfor %}
-                                    

-                                
-                            
-                        {% endfor %}
-
-                        
-                    
                 

             {% endif %}
 
@@ -331,7 +349,7 @@
         $('#div_edit_mails').hide();
         $('#div_edit_tags').hide();
         $('#div_edit_description').hide();
-        $("#page-Decoded").addClass("active");
+        $("#page-Tracker").addClass("active");
 
         $('#date-range-from').dateRangePicker({
             separator: ' to ',
diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py
index 3df13f00..e233bb00 100644
--- a/var/www/modules/restApi/Flask_restApi.py
+++ b/var/www/modules/restApi/Flask_restApi.py
@@ -22,8 +22,6 @@ from lib.objects import Items
 from lib import Tag
 from lib import Tracker
 
-from packages import Term  # TODO REMOVE ME
-
 from packages import Import_helper
 
 from importer.FeederImporter import api_add_json_feeder_to_queue
@@ -324,9 +322,10 @@ def get_all_tags():
     res = {'tags': Tag.get_all_tags()}
     return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
 
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-# # # # # # # # # # # # # #        TRACKER       # # # # # # # # # # # # # # # # #
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
+# # # # # # # # # # # # # #        TRACKER      # # # # # # # # # # # # # # # # # TODO
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
+'''
 @restApi.route("api/v1/add/tracker", methods=['POST'])
 @token_required('analyst')
 def add_tracker_term():
@@ -371,8 +370,21 @@ def get_tracker_metadata_api():
     data = request.get_json()
     tracker_uuid = data.get('tracker_uuid', None)
     req_data = {'tracker_uuid': tracker_uuid}
+    
+    tracker_uuid = request_dict.get('tracker_uuid', None)
+    if not request_dict:
+        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+    if not tracker_uuid:
+        return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
+    if not is_valid_uuid_v4(tracker_uuid):
+        return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
+    if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
+        return {'status': 'error', 'reason': 'Tracker not found'}, 404
+        
     res = Tracker.get_tracker_metadata_api(req_data)
     return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
+    
+'''
 
 '''
 
@@ -530,27 +542,27 @@ def get_domain_status_minimal():
     res[0]['domain'] = domain
     return create_json_response(res[0], res[1])
 
-@restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
-@token_required('analyst')
-def get_crawled_domain_list():
-    data = request.get_json()
-    res = get_mandatory_fields(data, ['date_from', 'date_to'])
-    if res:
-        return create_json_response(res[0], res[1])
-
-    date_from = data.get('date_from', None)
-    date_to = data.get('date_to', None)
-    domain_type = data.get('domain_type', None)
-    domain_status = 'UP'
-    # TODO TO MIGRATE
-    raise Exception('TO MIGRATE')
-    # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
-    dict_res = res[0]
-    dict_res['date_from'] = date_from
-    dict_res['date_to'] = date_to
-    dict_res['domain_status'] = domain_status
-    dict_res['domain_type'] = domain_type
-    return create_json_response(dict_res, res[1])
+# @restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
+# @token_required('analyst')
+# def get_crawled_domain_list():
+#     data = request.get_json()
+#     res = get_mandatory_fields(data, ['date_from', 'date_to'])
+#     if res:
+#         return create_json_response(res[0], res[1])
+#
+#     date_from = data.get('date_from', None)
+#     date_to = data.get('date_to', None)
+#     domain_type = data.get('domain_type', None)
+#     domain_status = 'UP'
+#     # TODO TO MIGRATE
+#     raise Exception('TO MIGRATE')
+#     # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
+#     dict_res = res[0]
+#     dict_res['date_from'] = date_from
+#     dict_res['date_to'] = date_to
+#     dict_res['domain_status'] = domain_status
+#     dict_res['domain_type'] = domain_type
+#     return create_json_response(dict_res, res[1])
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # #        IMPORT     # # # # # # # # # # # # # # # # # #
diff --git a/var/www/templates/hunter/menu_sidebar.html b/var/www/templates/hunter/menu_sidebar.html
index d727810e..d1959702 100644
--- a/var/www/templates/hunter/menu_sidebar.html
+++ b/var/www/templates/hunter/menu_sidebar.html
@@ -8,43 +8,43 @@