chg: [trackers] refactor trackers: track objects + filter by object types/sources/subtypes

bin/DB_KVROCKS_MIGRATION.py

@@ -233,17 +233,61 @@ def ail_2_ail_migration():
     # item in queue
     ail_2_ail.set_last_updated_sync_config()
 
+###############################
+#                             #
+#      TRACKER MIGRATION      #
+#                             #
+###############################
+
+def get_tracker_level(tracker_uuid):
+    level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level')
+    if not level:
+        level = 0
+    return int(level)
+
+def get_tracker_metadata(tracker_uuid):
+    meta = {'uuid': tracker_uuid,
+            'tracked': r_serv_tracker.hget('tracker:{tracker_uuid}', 'tracked'),
+            'type': r_serv_tracker.hget('tracker:{tracker_uuid}', 'type'),
+            'date': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date'),
+            'first_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen'),
+            'last_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen'),
+            'user_id': r_serv_tracker.hget('tracker:{tracker_uuid}', 'user_id'),
+            'level': get_tracker_level(tracker_uuid),
+            'mails': list(r_serv_tracker.smembers('tracker:mail:{tracker_uuid}')),
+            'sources': list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')),
+            'tags': list(r_serv_tracker.smembers(f'tracker:tags:{tracker_uuid}')),
+            'description': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'description'),
+            'webhook': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')}
+    return meta
+
+def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
+    all_item_id = set()
+    if date_from and date_to:
+        l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True)
+        if l_date_match:
+            dict_date_match = dict(l_date_match)
+            for date_day in Date.substract_date(date_from, date_to):
+                if date_day in dict_date_match:
+                    all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
+    return all_item_id
+
 # trackers + retro_hunts
 def trackers_migration():
     print('TRACKERS MIGRATION...')
     for tracker_uuid in old_Tracker.get_all_tracker_uuid():
-        meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True)
+        meta = get_tracker_metadata(tracker_uuid)
-        Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen'])
+        Tracker._re_create_tracker(meta['type'], meta['uuid'], meta['tracked'], meta['user_id'], meta['level'],
+                                   tags=meta['tags'], mails=meta['mails'], description=meta['description'],
+                                   webhook=meta['webhook'], sources=meta['sources'],
+                                   first_seen=meta['first_seen'], last_seen=meta['last_seen'])
 
+        tracker = Tracker.Tracker(tracker_uuid)
         # object migration # # TODO: in background
         for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']):
             print(item_id)
-            Tracker.add_tracked_item(tracker_uuid, item_id)
+            item_date = get_item_date(item_id)
+            tracker.add('item', '', item_id, date=item_date)
 
     print('RETRO HUNT MIGRATION...')
 
@@ -929,13 +973,13 @@ if __name__ == '__main__':
     # user_migration()
     #tags_migration()
     # items_migration()
-    crawler_migration()
+    # crawler_migration()
     # domain_migration()                      # TO TEST ###########################
     # decodeds_migration()
     # screenshots_migration()
     # subtypes_obj_migration()
     # ail_2_ail_migration()
-    # trackers_migration()
+    trackers_migration()
     # investigations_migration()
     ## statistics_migration()
 

bin/core/DbCleaner.py

@@ -14,23 +14,9 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from packages import Date
-from packages import Term
 
 from pubsublogger import publisher
 
-def clean_term_db_stat_token():
-    all_stat_date = Term.get_all_token_stat_history()
-
-    list_date_to_keep = Date.get_date_range(31)
-    for date in all_stat_date:
-        if date not in list_date_to_keep:
-            # remove history
-            Term.delete_token_statistics_by_date(date)
-
-    print('Term Stats Cleaned')
-
-
 if __name__ == "__main__":
 
     publisher.port = 6380
@@ -46,7 +32,7 @@ if __name__ == "__main__":
     while True:
 
         if daily_cleaner:
-            clean_term_db_stat_token()
+
             daily_cleaner = False
         else:
             sys.exit(0)

bin/exporter/MailExporter.py

@@ -105,7 +105,7 @@ class MailExporterTracker(MailExporter):
 
     def export(self, tracker, obj):  # TODO match
         tracker_type = tracker.get_type()
-        tracker_name = tracker.get_tracker()
+        tracker_name = tracker.get_tracked()
         subject = f'AIL Framework Tracker: {tracker_name}'  # TODO custom subject
         body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
         body += f'Item: {obj.id}\nurl:{obj.get_link()}'

bin/exporter/WebHookExporter.py

@@ -53,7 +53,7 @@ class WebHookExporterTracker(WebHookExporter):
         data = {'trackerId': tracker.get_uuid(),
                 'trackerType': tracker.get_type(),
                 'tags': tracker.get_tags(),
-                'tracker': tracker.get_tracker(),
+                'tracker': tracker.get_tracked(),
                 # object
                 'itemId': obj.get_id(),
                 'itemURL': obj.get_link()}

bin/lib/Investigations.py

@@ -186,9 +186,9 @@ class Investigation(object):
     def set_threat_level(self, threat_level):
         try:
             threat_level = int(threat_level)
-        except:
+        except TypeError:
             raise UpdateInvestigationError('threat_level Not an integer')
-        if threat_level >= 1 and threat_level <= 4:
+        if 1 <= threat_level <= 4:
             r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
         else:
             raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
@@ -196,9 +196,9 @@ class Investigation(object):
     def set_analysis(self, analysis):
         try:
             analysis = int(analysis)
-        except:
+        except TypeError:
             raise UpdateInvestigationError('analysis Not an integer')
-        if analysis >= 0 and analysis <= 2:
+        if 0 <= analysis <= 2:
             r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
         else:
             raise UpdateInvestigationError(f'Invalid analysis: {analysis}')

bin/lib/ail_core.py

@@ -40,6 +40,9 @@ def get_object_all_subtypes(obj_type):
         return ['telegram', 'twitter', 'jabber']
     return []
 
+def get_objects_tracked():
+    return ['decoded', 'item', 'pgp']
+
 def get_all_objects_with_subtypes_tuple():
     str_objs = []
     for obj_type in get_all_objects():

bin/lib/module_extractor.py

@@ -95,17 +95,18 @@ def convert_byte_offset_to_string(b_content, offset):
 def get_tracker_match(obj_id, content):
     extracted = []
     extracted_yara = []
-    trackers = Tracker.get_obj_all_trackers('item', '', obj_id)
+    trackers = Tracker.get_obj_trackers('item', '', obj_id)
     for tracker_uuid in trackers:
-        tracker_type = Tracker.get_tracker_type(tracker_uuid)
+        tracker = Tracker.Tracker(tracker_uuid)
+        tracker_type = tracker.get_type()
         # print(tracker_type)
-        tracker = Tracker.get_tracker_by_uuid(tracker_uuid)
+        tracked = tracker.get_tracked()
         if tracker_type == 'regex':  # TODO Improve word detection -> word delimiter
-            regex_match = regex_helper.regex_finditer(r_key, tracker, obj_id, content)
+            regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content)
             for match in regex_match:
-                extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
+                extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
         elif tracker_type == 'yara':
-            rule = Tracker.get_yara_rule_by_uuid(tracker_uuid)
+            rule = tracker.get_rule()
             rule.match(data=content.encode(), callback=_get_yara_match,
                        which_callbacks=yara.CALLBACK_MATCHES, timeout=30)
             yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}')
@@ -113,20 +114,20 @@ def get_tracker_match(obj_id, content):
             extracted = []
             for match in yara_match:
                 start, end, value = match.split(':', 2)
-                extracted_yara.append([int(start), int(end), value, f'tracker:{tracker_uuid}'])
+                extracted_yara.append([int(start), int(end), value, f'tracker:{tracker.uuid}'])
 
         elif tracker_type == 'word' or tracker_type == 'set':
             if tracker_type == 'set':
-                tracker = tracker.rsplit(';', 1)[0]
+                tracked = tracked.rsplit(';', 1)[0]
-                words = tracker.split(',')
+                words = tracked.split(',')
             else:
-                words = [tracker]
+                words = [tracked]
             for word in words:
                 regex = _get_word_regex(word)
                 regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content)
                 # print(regex_match)
                 for match in regex_match:
-                    extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
+                    extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
 
     # Convert byte offset to string offset
     if extracted_yara:

bin/lib/objects/CryptoCurrencies.py

@@ -2,12 +2,13 @@
 # -*-coding:UTF-8 -*
 
 import os
+import re
 import sys
 
 from flask import url_for
 from hashlib import sha256
 
-from pymisp import MISPObject, MISPAttribute
+from pymisp import MISPObject
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
@@ -175,31 +176,57 @@ def get_all_cryptocurrencies():
         cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype)
     return cryptos
 
-
 def get_all_cryptocurrencies_by_subtype(subtype):
     return get_all_id('cryptocurrency', subtype)
 
+def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
+    if subtype == '':
+        pass
+    elif subtype == 'name':
+        pass
+    elif subtype == 'mail':
+        pass
+    return name_to_search
 
-# TODO save object
+def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
-def import_misp_object(misp_obj):
+    cryptocurrencies = {}
-    """
+    # for subtype in subtypes:
-    :type misp_obj: MISPObject
+    r_name = sanitize_cryptocurrency_name_to_search(name_to_search, subtype)
-    """
+    if not name_to_search or isinstance(r_name, dict):
-    obj_id = None
+        # break
-    obj_subtype = None
+        return cryptocurrencies
-    for attribute in misp_obj.attributes:
+    r_name = re.compile(r_name)
-        if attribute.object_relation == 'address':  # TODO: handle xmr address field
+    for crypto_name in get_all_cryptocurrencies_by_subtype(subtype):
-            obj_id = attribute.value
+        res = re.search(r_name, crypto_name)
-        elif attribute.object_relation == 'symbol':
+        if res:
-            obj_subtype = get_subtype_by_symbol(attribute.value)
+            cryptocurrencies[crypto_name] = {}
-    if obj_id and obj_subtype:
+            if r_pos:
-        obj = CryptoCurrency(obj_id, obj_subtype)
+                cryptocurrencies[crypto_name]['hl-start'] = res.start()
-        first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
+                cryptocurrencies[crypto_name]['hl-end'] = res.end()
-        tags = obj.get_misp_object_tags(misp_obj)
+    return cryptocurrencies
-        # for tag in tags:
-        #     obj.add_tag()
 
 
-if __name__ == '__main__':
+# # TODO save object
-    res = get_all_cryptocurrencies()
+# def import_misp_object(misp_obj):
-    print(res)
+#     """
+#     :type misp_obj: MISPObject
+#     """
+#     obj_id = None
+#     obj_subtype = None
+#     for attribute in misp_obj.attributes:
+#         if attribute.object_relation == 'address':  # TODO: handle xmr address field
+#             obj_id = attribute.value
+#         elif attribute.object_relation == 'symbol':
+#             obj_subtype = get_subtype_by_symbol(attribute.value)
+#     if obj_id and obj_subtype:
+#         obj = CryptoCurrency(obj_id, obj_subtype)
+#         first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
+#         tags = obj.get_misp_object_tags(misp_obj)
+#         # for tag in tags:
+#         #     obj.add_tag()
+
+
+# if __name__ == '__main__':
+#     name_to_search = '3c'
+#     subtype = 'bitcoin'
+#     print(search_cryptocurrency_by_name(name_to_search, subtype))

bin/lib/objects/Cves.py

@@ -2,9 +2,11 @@
 # -*-coding:UTF-8 -*
 
 import os
+import re
 import sys
 
 from flask import url_for
+
 from pymisp import MISPObject
 
 import requests
@@ -110,6 +112,25 @@ def get_cves_meta(cves_id, options=set()):
         dict_cve[cve_id] = cve.get_meta(options=options)
     return dict_cve
 
+def sanitize_cve_name_to_search(name_to_search): # TODO FILTER NAME
+    return name_to_search
+
+def search_cves_by_name(name_to_search, r_pos=False):
+    cves = {}
+    # for subtype in subtypes:
+    r_name = sanitize_cve_name_to_search(name_to_search)
+    if not name_to_search or isinstance(r_name, dict):
+        return cves
+    r_name = re.compile(r_name)
+    for cve_name in get_all_cves():
+        res = re.search(r_name, cve_name)
+        if res:
+            cves[cve_name] = {}
+            if r_pos:
+                cves[cve_name]['hl-start'] = res.start()
+                cves[cve_name]['hl-end'] = res.end()
+    return cves
+
 def api_get_cves_range_by_daterange(date_from, date_to):
     cves = []
     for date in Date.substract_date(date_from, date_to):
@@ -133,3 +154,5 @@ def get_cve_graphline(cve_id):
 
 
 # if __name__ == '__main__':
+#     name_to_search = '98'
+#     print(search_cves_by_name(name_to_search))

bin/lib/objects/Decodeds.py

@@ -2,6 +2,7 @@
 # -*-coding:UTF-8 -*
 
 import os
+import re
 import sys
 import magic
 import requests
@@ -9,6 +10,7 @@ import zipfile
 
 from flask import url_for
 from io import BytesIO
+
 from pymisp import MISPObject
 
 sys.path.append(os.environ['AIL_BIN'])
@@ -114,11 +116,15 @@ class Decoded(AbstractDaterangeObject):
     def get_filepath(self, mimetype=None):
         return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype))
 
-    def get_content(self, mimetype=None):
+    def get_content(self, mimetype=None, r_str=False):
         filepath = self.get_filepath(mimetype=mimetype)
+        if r_str:
+            with open(filepath, 'r') as f:
+                content = f.read()
+        else:
             with open(filepath, 'rb') as f:
-            file_content = BytesIO(f.read())
+                content = BytesIO(f.read())
-        return file_content
+        return content
 
     def get_zip_content(self):
         # mimetype = self.get_estimated_type()
@@ -347,6 +353,25 @@ def sanitise_mimetype(mimetype):
         else:
             return None
 
+def sanitize_decoded_name_to_search(name_to_search): # TODO FILTER NAME
+    return name_to_search
+
+def search_decodeds_by_name(name_to_search, r_pos=False):
+    decodeds = {}
+    # for subtype in subtypes:
+    r_name = sanitize_decoded_name_to_search(name_to_search)
+    if not name_to_search or isinstance(r_name, dict):
+        return decodeds
+    r_name = re.compile(r_name)
+    for decoded_name in get_all_decodeds():
+        res = re.search(r_name, decoded_name)
+        if res:
+            decodeds[decoded_name] = {}
+            if r_pos:
+                decodeds[decoded_name]['hl-start'] = res.start()
+                decodeds[decoded_name]['hl-end'] = res.end()
+    return decodeds
+
 ############################################################################
 
 def sanityze_decoder_names(decoder_name):
@@ -512,4 +537,7 @@ def get_all_decodeds_files():
             decodeds.append(file)
     return decodeds
 
+
 # if __name__ == '__main__':
+#     name_to_search = '4d36'
+#     print(search_decodeds_by_name(name_to_search))

bin/lib/objects/Domains.py

@@ -96,6 +96,9 @@ class Domain(AbstractObject):
         elif int(last_check) < date:
             self._set_last_check(date)
 
+    def get_content(self):
+        return self.id
+
     def get_last_origin(self, obj=False):
         origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')}
         if obj and origin['item']:

bin/lib/objects/Items.py

@@ -79,7 +79,7 @@ class Item(AbstractObject):
         else:
             return filename
 
-    def get_content(self, binary=False):
+    def get_content(self, r_str=True, binary=False):
         """
         Returns Item content
         """

bin/lib/objects/Pgps.py

@@ -2,6 +2,7 @@
 # -*-coding:UTF-8 -*
 
 import os
+import re
 import sys
 
 from flask import url_for
@@ -100,5 +101,35 @@ def get_all_pgps():
 def get_all_pgps_by_subtype(subtype):
     return get_all_id('pgp', subtype)
 
+# TODO FILTER NAME + Key + mail
+def sanitize_pgp_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
+    if subtype == 'key':
+        pass
+    elif subtype == 'name':
+        pass
+    elif subtype == 'mail':
+        pass
+    return name_to_search
+
+def search_pgps_by_name(name_to_search, subtype, r_pos=False):
+    pgps = {}
+    # for subtype in subtypes:
+    r_name = sanitize_pgp_name_to_search(name_to_search, subtype)
+    if not name_to_search or isinstance(r_name, dict):
+        # break
+        return pgps
+    r_name = re.compile(r_name)
+    for pgp_name in get_all_pgps_by_subtype(subtype):
+        res = re.search(r_name, pgp_name)
+        if res:
+            pgps[pgp_name] = {}
+            if r_pos:
+                pgps[pgp_name]['hl-start'] = res.start()
+                pgps[pgp_name]['hl-end'] = res.end()
+    return pgps
+
 
 # if __name__ == '__main__':
+#     name_to_search = 'ex'
+#     subtype = 'name'
+#     print(search_pgps_by_name(name_to_search, subtype))

bin/lib/objects/Screenshots.py

@@ -3,6 +3,7 @@
 
 import base64
 import os
+import re
 import sys
 
 from hashlib import sha256
@@ -71,6 +72,9 @@ class Screenshot(AbstractObject):
             file_content = BytesIO(f.read())
         return file_content
 
+    def get_content(self):
+        return self.get_file_content()
+
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('file')
@@ -129,4 +133,26 @@ def create_screenshot(content, size_limit=5000000, b64=True, force=False):
         return screenshot
     return  None
 
+def sanitize_screenshot_name_to_search(name_to_search): # TODO FILTER NAME
+    return name_to_search
+
+def search_screenshots_by_name(name_to_search, r_pos=False):
+    screenshots = {}
+    # for subtype in subtypes:
+    r_name = sanitize_screenshot_name_to_search(name_to_search)
+    if not name_to_search or isinstance(r_name, dict):
+        return screenshots
+    r_name = re.compile(r_name)
+    for screenshot_name in get_all_screenshots():
+        res = re.search(r_name, screenshot_name)
+        if res:
+            screenshots[screenshot_name] = {}
+            if r_pos:
+                screenshots[screenshot_name]['hl-start'] = res.start()
+                screenshots[screenshot_name]['hl-end'] = res.end()
+    return screenshots
+
+
 # if __name__ == '__main__':
+#     name_to_search = '29ba'
+#     print(search_screenshots_by_name(name_to_search))

bin/lib/objects/Usernames.py

@@ -3,13 +3,11 @@
 
 import os
 import sys
-import redis
+import re
 
 from flask import url_for
 from pymisp import MISPObject
 
-# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
@@ -107,9 +105,30 @@ def get_all_usernames():
 def get_all_usernames_by_subtype(subtype):
     return get_all_id('username', subtype)
 
+# TODO FILTER NAME + Key + mail
+def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
+
+    return name_to_search
+
+def search_usernames_by_name(name_to_search, subtype, r_pos=False):
+    usernames = {}
+    # for subtype in subtypes:
+    r_name = sanitize_username_name_to_search(name_to_search, subtype)
+    if not name_to_search or isinstance(r_name, dict):
+        # break
+        return usernames
+    r_name = re.compile(r_name)
+    for user_name in get_all_usernames_by_subtype(subtype):
+        res = re.search(r_name, user_name)
+        if res:
+            usernames[user_name] = {}
+            if r_pos:
+                usernames[user_name]['hl-start'] = res.start()
+                usernames[user_name]['hl-end'] = res.end()
+    return usernames
 
 
-if __name__ == '__main__':
+# if __name__ == '__main__':
-
+#     name_to_search = 'co'
-    obj = Username('ninechantw', 'telegram')
+#     subtype = 'telegram'
-    print(obj.get_misp_object().to_json())
+#     print(search_usernames_by_name(name_to_search, subtype))

bin/lib/objects/abstract_daterange_object.py

@@ -108,6 +108,9 @@ class AbstractDaterangeObject(AbstractObject, ABC):
             sparkline.append(self.get_nb_seen_by_date(date))
         return sparkline
 
+    def get_content(self):
+        return self.id
+
     def _add_create(self):
         r_object.sadd(f'{self.type}:all', self.id)
 

bin/lib/objects/abstract_object.py

@@ -19,9 +19,9 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib import Tag
 from lib import Duplicate
-from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
+from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
-from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
+from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 
 class AbstractObject(ABC):
@@ -85,6 +85,13 @@ class AbstractObject(ABC):
 
     #- Tags -#
 
+    @abstractmethod
+    def get_content(self):
+        """
+        Get Object Content
+        """
+        pass
+
     ## Duplicates ##
     def get_duplicates(self):
         return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id)
@@ -125,7 +132,7 @@ class AbstractObject(ABC):
         return is_obj_tracked(self.type, self.subtype, self.id)
 
     def get_trackers(self):
-        return get_obj_all_trackers(self.type, self.subtype, self.id)
+        return get_obj_trackers(self.type, self.subtype, self.id)
 
     def delete_trackers(self):
         return delete_obj_trackers(self.type, self.subtype, self.id)
@@ -134,13 +141,14 @@ class AbstractObject(ABC):
 
     def _delete(self):
         # DELETE TAGS
-        Tag.delete_obj_all_tags(self.id, self.type)  # ########### # TODO: # TODO: # FIXME:
+        Tag.delete_object_tags(self.type, self.get_subtype(r_str=True), self.id)
         # remove from tracker
         self.delete_trackers()
+        # remove from retro hunt currently item only TODO
         # remove from investigations
         self.delete_investigations()
-
+        # Delete Correlations
-        # # TODO: remove from correlation
+        delete_obj_correlations(self.type, self.get_subtype(r_str=True), self.id)
 
     @abstractmethod
     def delete(self):

bin/lib/objects/abstract_subtype_object.py

@@ -116,6 +116,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
             if date > last_seen:
                 self.set_last_seen(date)
 
+    def get_content(self):
+        return self.id
+
     def get_sparkline(self):
         sparkline = []
         for date in Date.get_previous_date_list(6):

bin/lib/objects/ail_objects.py

@@ -138,9 +138,14 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
 
 def get_objects_meta(objs, options=set(), flask_context=False):
     metas = []
-    for obj_dict in objs:
+    for obj in objs:
-        metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options,
+        if isinstance(obj, dict):
-                                     flask_context=flask_context))
+            obj_type = obj['type']
+            subtype = obj['subtype']
+            obj_id = obj['id']
+        else:
+            obj_type, subtype, obj_id = obj.split(':', 2)
+        metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
     return metas
 
 
@@ -162,7 +167,25 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
     return meta
 
 
-def get_ui_obj_tag_table_keys(obj_type):
+#### OBJ FILTERS ####
+
+def is_filtered(obj, filters):
+    if 'mimetypes' in filters:
+        mimetype = obj.get_mimetype()
+        if mimetype not in filters['mimetypes']:
+            return True
+    if 'sources' in filters:
+        obj_source = obj.get_source()
+        if obj_source not in filters['sources']:
+            return True
+    if 'subtypes' in filters:
+        subtype = obj.get_subtype(r_str=True)
+        if subtype not in filters['subtypes']:
+            return True
+    return False
+
+
+def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
     """
     Warning: use only in flask (dynamic templates)
     """
@@ -170,6 +193,7 @@ def get_ui_obj_tag_table_keys(obj_type):
         return ['id', 'first_seen', 'last_check', 'status']  # # TODO: add root screenshot
 
 
+
 # # # # MISP OBJECTS # # # #
 
 # # TODO: CHECK IF object already have an UUID
@@ -265,6 +289,7 @@ def api_sanitize_object_type(obj_type):
     if not is_valid_object_type(obj_type):
         return {'status': 'error', 'reason': 'Incorrect object type'}, 400
 
+#### CORRELATION ####
 
 def get_obj_correlations(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
@@ -292,13 +317,14 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0,
 
 def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
     objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
+    # print(objs)
     for obj_tuple in objs:
         obj1_type, subtype1, id1 = obj_tuple
         add_obj_tags(obj1_type, subtype1, id1, tags)
     return objs
 
 ################################################################################
-################################################################################
+################################################################################ TODO
 ################################################################################
 
 def delete_obj_correlations(obj_type, subtype, obj_id):
@@ -357,7 +383,7 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
             "links": create_correlation_graph_links(links)}
 
 
-###############
+# --- CORRELATION --- #
 
 
 # if __name__ == '__main__':

bin/modules/Decoder.py

@@ -23,6 +23,9 @@ from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
 from lib.objects.Items import Item
 from lib.objects.Decodeds import Decoded
+from trackers.Tracker_Term import Tracker_Term
+from trackers.Tracker_Regex import Tracker_Regex
+from trackers.Tracker_Yara import Tracker_Yara
 
 config_loader = ConfigLoader()
 hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal")
@@ -76,6 +79,10 @@ class Decoder(AbstractModule):
         # Waiting time in seconds between to message processed
         self.pending_seconds = 1
 
+        self.tracker_term = Tracker_Term(queue=False)
+        self.tracker_regex = Tracker_Regex(queue=False)
+        self.tracker_yara = Tracker_Yara(queue=False)
+
         # Send module state to logs
         self.redis_logger.info(f'Module {self.module_name} initialized')
 
@@ -84,6 +91,7 @@ class Decoder(AbstractModule):
         item = Item(message)
         content = item.get_content()
         date = item.get_date()
+        new_decodeds = []
 
         for decoder in self.decoder_order:
             find = False
@@ -96,8 +104,8 @@ class Decoder(AbstractModule):
             encodeds = set(encodeds)
 
             for encoded in encodeds:
-                find = False
                 if len(encoded) >= decoder['encoded_min_size']:
+                    find = True
                     decoded_file = self.decoder_function[dname](encoded)
 
                     sha1_string = sha1(decoded_file).hexdigest()
@@ -109,6 +117,7 @@ class Decoder(AbstractModule):
                             print(sha1_string, item.id)
                             raise Exception(f'Invalid mimetype: {decoded.id} {item.id}')
                         decoded.save_file(decoded_file, mimetype)
+                        new_decodeds.append(decoded.id)
                     else:
                         mimetype = decoded.get_mimetype()
                     decoded.add(dname, date, item.id, mimetype=mimetype)
@@ -125,6 +134,13 @@ class Decoder(AbstractModule):
                 msg = f'infoleak:automatic-detection="{dname}";{item.id}'
                 self.add_message_to_queue(msg, 'Tags')
 
+                ####################
+                # TRACKERS DECODED
+                for decoded_id in new_decodeds:
+                    self.tracker_term.compute(decoded_id, obj_type='decoded')
+                    self.tracker_regex.compute(decoded_id, obj_type='decoded')
+                    self.tracker_yara.compute(decoded_id, obj_type='decoded')
+
 
 if __name__ == '__main__':
     module = Decoder()

bin/modules/PgpDump.py

@@ -216,16 +216,16 @@ class PgpDump(AbstractModule):
                 pgp = Pgps.Pgp(name, 'name')
                 pgp.add(date, self.item_id)
                 print(f'    name: {name}')
-                self.tracker_term.compute(self.item_id, item_content=name)
+                self.tracker_term.compute(name, obj_type='pgp', subtype='name')
-                self.tracker_regex.compute(self.item_id, content=name)
+                self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
-                self.tracker_yara.compute(self.item_id, item_content=name)
+                self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
             for mail in self.mails:
                 pgp = Pgps.Pgp(mail, 'mail')
                 pgp.add(date, self.item_id)
                 print(f'    mail: {mail}')
-                self.tracker_term.compute(self.item_id, item_content=mail)
+                self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
-                self.tracker_regex.compute(self.item_id, content=mail)
+                self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')
-                self.tracker_yara.compute(self.item_id, item_content=mail)
+                self.tracker_yara.compute(mail, obj_type='pgp', subtype='mail')
 
             # Keys extracted from PGP PRIVATE KEY BLOCK
             for key in self.private_keys:

bin/packages/Term.py

@@ -1,532 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import re
-import sys
-import time
-import uuid
-import datetime
-
-from collections import defaultdict
-from flask import escape
-
-from nltk.tokenize import RegexpTokenizer
-from textblob import TextBlob
-
-sys.path.append(os.environ['AIL_BIN'])
-##################################
-# Import Project packages
-##################################
-from lib import ConfigLoader
-from lib import Tracker
-from packages import Date
-from lib.objects import Items
-
-config_loader = ConfigLoader.ConfigLoader()
-r_serv_term = config_loader.get_db_conn("Kvrocks_Trackers")
-config_loader = None
-
-email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
-email_regex = re.compile(email_regex)
-
-special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
-special_characters.add('\\s')
-
-# NLTK tokenizer
-tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
-                                    gaps=True, discard_empty=True)
-
-def is_valid_uuid_v4(UUID):
-    if not UUID:
-        return False
-    UUID = UUID.replace('-', '')
-    try:
-        uuid_test = uuid.UUID(hex=UUID, version=4)
-        return uuid_test.hex == UUID
-    except:
-        return False
-
-# # TODO: use new package => duplicate fct
-def is_in_role(user_id, role):
-    if r_serv_term.sismember('user_role:{}'.format(role), user_id):
-        return True
-    else:
-        return False
-
-def check_term_uuid_valid_access(term_uuid, user_id):
-    if not is_valid_uuid_v4(term_uuid):
-        return {"status": "error", "reason": "Invalid uuid"}, 400
-    level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
-    if not level:
-        return {"status": "error", "reason": "Unknown uuid"}, 404
-    if level == 0:
-        if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
-            if not is_in_role(user_id, 'admin'):
-                return {"status": "error", "reason": "Unknown uuid"}, 404
-    return None
-
-
-def is_valid_mail(email):
-    result = email_regex.match(email)
-    if result:
-        return True
-    else:
-        return False
-
-def verify_mail_list(mail_list):
-    for mail in mail_list:
-        if not is_valid_mail(mail):
-            return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
-    return None
-
-def is_valid_regex(term_regex):
-    try:
-        re.compile(term_regex)
-        return True
-    except:
-        return False
-
-def get_text_word_frequency(item_content, filtering=True):
-    item_content = item_content.lower()
-    words_dict = defaultdict(int)
-
-    if filtering:
-        blob = TextBlob(item_content, tokenizer=tokenizer)
-    else:
-        blob = TextBlob(item_content)
-    for word in blob.tokens:
-        words_dict[word] += 1
-    return words_dict
-
-# # TODO: create all tracked words
-def get_tracked_words_list():
-    return list(r_serv_term.smembers('all:tracker:word'))
-
-def get_set_tracked_words_list():
-    set_list = r_serv_term.smembers('all:tracker:set')
-    all_set_list = []
-    for elem in set_list:
-        res = elem.split(';')
-        num_words = int(res[1])
-        ter_set = res[0].split(',')
-        all_set_list.append((ter_set, num_words, elem))
-    return all_set_list
-
-def get_regex_tracked_words_dict():
-    regex_list = r_serv_term.smembers('all:tracker:regex')
-    dict_tracked_regex = {}
-    for regex in regex_list:
-        dict_tracked_regex[regex] = re.compile(regex)
-    return dict_tracked_regex
-
-def get_tracked_term_list_item(term_uuid, date_from, date_to):
-    all_item_id = []
-    if date_from and date_to:
-        for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)):
-            all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date)))
-    return all_item_id
-
-def is_term_tracked_in_global_level(term, term_type):
-    res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))
-    if res:
-        for elem_uuid in res:
-            if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level') == '1':
-                return True
-    return False
-
-def is_term_tracked_in_user_level(term, term_type, user_id):
-    res = r_serv_term.smembers('user:tracker:{}'.format(user_id))
-    if res:
-        for elem_uuid in res:
-            if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked') == term:
-                if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type') == term_type:
-                    return True
-    return False
-
-def parse_json_term_to_add(dict_input, user_id):
-    term = dict_input.get('term', None)
-    if not term:
-        return {"status": "error", "reason": "Term not provided"}, 400
-    term_type = dict_input.get('type', None)
-    if not term_type:
-        return {"status": "error", "reason": "Term type not provided"}, 400
-
-    nb_words = dict_input.get('nb_words', 1)
-
-    description = dict_input.get('description', '')
-    description = escape(description)
-
-    webhook = dict_input.get('webhook', '')
-    webhook = escape(webhook)
-
-    res = parse_tracked_term_to_add(term, term_type, nb_words=nb_words)
-    if res[1] != 200:
-        return res
-    term = res[0]['term']
-    term_type = res[0]['type']
-
-    tags = dict_input.get('tags', [])
-    mails = dict_input.get('mails', [])
-    res = verify_mail_list(mails)
-    if res:
-        return res
-
-    level = dict_input.get('level', 1)
-    try:
-        level = int(level)
-        if level not in range(0, 1):
-            level = 1
-    except:
-        level = 1
-
-    # check if term already tracked in global
-    if level == 1:
-        if is_term_tracked_in_global_level(term, term_type):
-            return {"status": "error", "reason": "Term already tracked"}, 409
-    else:
-        if is_term_tracked_in_user_level(term, term_type, user_id):
-            return {"status": "error", "reason": "Term already tracked"}, 409
-
-    term_uuid = add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook)
-
-    return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
-
-
-def parse_tracked_term_to_add(term, term_type, nb_words=1):
-    if term_type == 'regex':
-        if not is_valid_regex(term):
-            return {"status": "error", "reason": "Invalid regex"}, 400
-    elif term_type == 'word' or term_type == 'set':
-        # force lowercase
-        term = term.lower()
-        word_set = set(term)
-        set_inter = word_set.intersection(special_characters)
-        if set_inter:
-            return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
-        words = term.split()
-        # not a word
-        if term_type == 'word' and len(words) > 1:
-            term_type = 'set'
-
-        # output format: term1,term2,term3;2
-        if term_type == 'set':
-            try:
-                nb_words = int(nb_words)
-            except:
-                nb_words = 1
-            if nb_words == 0:
-                nb_words = 1
-
-            words_set = set(words)
-            words_set = sorted(words_set)
-
-            if nb_words > len(words_set):
-                nb_words = len(words_set)
-
-            term = ",".join(words_set)
-            term = "{};{}".format(term, nb_words)
-
-    elif term_type == 'yara_custom':
-        if not Tracker.is_valid_yara_rule(term):
-            return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
-    elif term_type == 'yara_default':
-        if not Tracker.is_valid_default_yara_rule(term):
-            return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
-    else:
-        return {"status": "error", "reason": "Incorrect type"}, 400
-    return {"status": "success", "term": term, "type": term_type}, 200
-
-def add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook, dashboard=0):
-
-    term_uuid = str(uuid.uuid4())
-
-    # YARA
-    if term_type == 'yara_custom' or term_type == 'yara_default':
-        term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
-        term_type = 'yara'
-
-    # create metadata
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked', term)  # # TODO: use hash
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level)
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard)
-
-    if description:
-        r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
-
-    if webhook:
-        r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
-
-    # create all term set
-    r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
-
-    # create term - uuid map
-    r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
-
-    # add display level set
-    if level == 0:  # user only
-        r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid)
-        r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
-    elif level == 1:  # global
-        r_serv_term.sadd('global:tracker', term_uuid)
-        r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid)
-
-    # create term tags list
-    for tag in tags:
-        r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), escape(tag))
-
-    # create term tags mail notification list
-    for mail in mails:
-        r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), escape(mail))
-
-    # toggle refresh module tracker list/set
-    r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
-
-    return term_uuid
-
-
-def parse_tracked_term_to_delete(dict_input, user_id):
-    term_uuid = dict_input.get("uuid", None)
-    res = check_term_uuid_valid_access(term_uuid, user_id)
-    if res:
-        return res
-    delete_term(term_uuid)
-    return {"uuid": term_uuid}, 200
-
-
-# # TODO: MOVE IN TRACKER
-def delete_term(term_uuid):
-    term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
-    term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
-    level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
-    r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
-
-    r_serv_term.srem(f'trackers:all', term_uuid)
-    r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
-
-    # Term not tracked by other users
-    if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
-        r_serv_term.srem('all:tracker:{}'.format(term_type), term)
-
-        # toggle refresh module tracker list/set
-        r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
-
-    if level == '0':  # user only
-        user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
-        r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid)
-        r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
-    elif level == '1':  # global
-        r_serv_term.srem('global:tracker', term_uuid)
-        r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid)
-
-    # delete metatadata
-    r_serv_term.delete('tracker:{}'.format(term_uuid))
-
-    # remove tags
-    r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
-
-    # remove mails
-    r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
-
-    # remove sources
-    r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
-
-    # remove item set
-    all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
-    if all_item_date:
-        all_item_date = dict(all_item_date)
-        for date in all_item_date:
-            for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
-                r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
-            r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
-        r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
-
-    if term_type == 'yara':
-        # delete custom rule
-        if not Tracker.is_default_yara_rule(term):
-            filepath = Tracker.get_yara_rule_file_by_tracker_name(term)
-            if filepath:
-                os.remove(filepath)
-
-def replace_tracker_description(term_uuid, description):
-    description = escape(description)
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
-
-def replace_tracked_term_tags(term_uuid, tags):
-    r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
-    for tag in tags:
-        tag = escape(tag)
-        r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag)
-
-def replace_tracked_term_mails(term_uuid, mails):
-    res = verify_mail_list(mails)
-    if res:
-        return res
-    else:
-        r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
-        for mail in mails:
-            mail = escape(mail)
-            r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail)
-
-def get_term_uuid_list(term, term_type):
-    return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)))
-
-def get_term_tags(term_uuid):
-    return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)))
-
-def get_term_mails(term_uuid):
-    return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)))
-
-def get_term_webhook(term_uuid):
-    return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook")
-
-def add_tracked_item(term_uuid, item_id, item_date):
-    # track item
-    r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id)
-    # track nb item by date
-    r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), {item_date: item_date})
-
-def create_token_statistics(item_date, word, nb):
-    r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), 1, word)
-    r_serv_term.zincrby('stat_token_total_by_day:{}'.format(item_date), nb, word)
-    r_serv_term.sadd('stat_token_history', item_date)
-
-def delete_token_statistics_by_date(item_date):
-    r_serv_term.delete('stat_token_per_item_by_day:{}'.format(item_date))
-    r_serv_term.delete('stat_token_total_by_day:{}'.format(item_date))
-    r_serv_term.srem('stat_token_history', item_date)
-
-def get_all_token_stat_history():
-    return r_serv_term.smembers('stat_token_history')
-
-def get_tracked_term_last_updated_by_type(term_type):
-    epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type))
-    if not epoch_update:
-        epoch_update = 0
-    return float(epoch_update)
-
-def parse_get_tracker_term_item(dict_input, user_id):
-    term_uuid = dict_input.get('uuid', None)
-    res = check_term_uuid_valid_access(term_uuid, user_id)
-    if res:
-        return res
-
-    date_from = dict_input.get('date_from', None)
-    date_to = dict_input.get('date_to', None)
-
-    if date_from is None:
-        date_from = get_tracked_term_first_seen(term_uuid)
-        if date_from:
-            date_from = date_from[0]
-
-    if date_to is None:
-        date_to = date_from
-
-    if date_from > date_to:
-        date_from = date_to
-
-    all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
-    all_item_id = Items.get_item_list_desc(all_item_id)
-
-    res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
-    return res_dict, 200
-
-def get_tracked_term_first_seen(term_uuid):
-    return Tracker.get_tracker_first_seen(term_uuid)
-
-def get_tracked_term_last_seen(term_uuid):
-    return Tracker.get_tracker_last_seen(term_uuid)
-
-def get_term_metedata(term_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False):
-    dict_uuid = {}
-    dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
-    dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
-    dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
-    dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
-    dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
-    if user_id:
-        dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
-    if level:
-        dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
-    if mails:
-        dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
-    if tags:
-        dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
-    if sparkline:
-        dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
-    if description:
-        dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
-    dict_uuid['uuid'] = term_uuid
-    return dict_uuid
-
-def get_tracked_term_sparkline(tracker_uuid, num_day=6):
-    date_range_sparkline = Date.get_date_range(num_day)
-    sparklines_value = []
-    for date_day in date_range_sparkline:
-        nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
-        if nb_seen_this_day is None:
-            nb_seen_this_day = 0
-        sparklines_value.append(int(nb_seen_this_day))
-    return sparklines_value
-
-def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None):
-    if date_from and date_to:
-        date_range = Date.substract_date(date_from, date_to)
-    else:
-        date_range = Date.get_date_range(num_day)
-    list_tracker_stats = []
-    for tracker_uuid in list_tracker_uuid:
-        dict_tracker_data = []
-        tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked')
-        for date_day in date_range:
-            nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
-            if nb_seen_this_day is None:
-                nb_seen_this_day = 0
-            dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
-        list_tracker_stats.append({"name": tracker, "Data": dict_tracker_data})
-    return list_tracker_stats
-
-def get_list_trackeed_term_tags(term_uuid):
-    res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))
-    if res:
-        return list(res)
-    else:
-        return []
-
-def get_list_trackeed_term_mails(term_uuid):
-    res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))
-    if res:
-        return list(res)
-    else:
-        return []
-
-def get_user_tracked_term_uuid(user_id, filter_type=None):
-    if filter_type:
-        return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id, filter_type)))
-    else:
-        return list(r_serv_term.smembers('user:tracker:{}'.format(user_id)))
-
-def get_global_tracked_term_uuid(filter_type=None):
-    if filter_type:
-        return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type)))
-    else:
-        return list(r_serv_term.smembers('global:tracker'))
-
-def get_all_user_tracked_terms(user_id, filter_type=None):
-    all_user_term = []
-    all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type)
-
-    for term_uuid in all_user_term_uuid:
-        all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True))
-    return all_user_term
-
-def get_all_global_tracked_terms(filter_type=None):
-    all_user_term = []
-    all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type)
-
-    for term_uuid in all_user_term_uuid:
-        all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True))
-    return all_user_term

bin/packages/lib_words.py

@@ -3,8 +3,6 @@
 import os
 import string
 
-from pubsublogger import publisher
-
 import calendar
 from datetime import date
 from dateutil.rrule import rrule, DAILY

bin/trackers/Retro_Hunt.py

@@ -61,15 +61,17 @@ class Retro_Hunt(AbstractModule):
         self.progress = 0
         # First launch
         # restart
-        rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
+        retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
 
-        timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
+        rule = retro_hunt.get_rule(r_compile=True)
+
+        timeout = retro_hunt.get_timeout()
         self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
-        sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
+        sources = retro_hunt.get_sources(r_sort=True)
 
-        self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
+        self.date_from = retro_hunt.get_date_from()
-        self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid)
+        self.date_to = retro_hunt.get_date_to()
-        self.tags = Tracker.get_retro_hunt_task_tags(task_uuid)
+        self.tags = retro_hunt.get_tags()
         curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
         self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
         self.update_progress(sources, curr_date)
@@ -106,11 +108,10 @@ class Retro_Hunt(AbstractModule):
 
                     # PAUSE
                     self.update_progress(sources, curr_date)
-                    if Tracker.check_retro_hunt_pause(task_uuid):
+                    if retro_hunt.to_pause():
                         Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
                         # self.update_progress(sources, curr_date, save_db=True)
-                        Tracker.pause_retro_hunt_task(task_uuid)
+                        retro_hunt.pause()
-                        Tracker.clear_retro_hunt_task_cache(task_uuid)
                         return None
 
                 self.nb_src_done += 1
@@ -120,9 +121,7 @@ class Retro_Hunt(AbstractModule):
 
         self.update_progress(sources, curr_date)
 
-        Tracker.set_retro_hunt_task_state(task_uuid, 'completed')
+        retro_hunt.complete()
-        Tracker.set_retro_hunt_nb_match(task_uuid)
-        Tracker.clear_retro_hunt_task_cache(task_uuid)
 
         print(f'Retro Hunt {task_uuid} completed')
         self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
@@ -130,10 +129,11 @@ class Retro_Hunt(AbstractModule):
         # # TODO: stop
 
     def update_progress(self, sources, curr_date, save_db=False):
-        progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to,
+        retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
+        progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
                                                sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
         if self.progress != progress:
-            Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress)
+            retro_hunt.set_progress(progress)
             self.progress = progress
         # if save_db:
         #     Tracker.set_retro_hunt_task_progress(task_uuid, progress)

bin/trackers/Tracker_Regex.py

@@ -17,9 +17,8 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
+from lib.objects import ail_objects
 from lib.ConfigLoader import ConfigLoader
-from packages import Term
 from lib import Tracker
 
 from exporter.MailExporter import MailExporterTracker
@@ -39,7 +38,7 @@ class Tracker_Regex(AbstractModule):
         self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time")
 
         # refresh Tracked Regex
-        self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
+        self.tracked_regexs = Tracker.get_tracked_regexs()
         self.last_refresh = time.time()
 
         # Exporter
@@ -48,56 +47,65 @@ class Tracker_Regex(AbstractModule):
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
-    def compute(self, item_id, content=None):
+    def compute(self, obj_id, obj_type='item', subtype=''):
         # refresh Tracked regex
         if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
-            self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
+            self.tracked_regexs = Tracker.get_tracked_regexs()
             self.last_refresh = time.time()
             self.redis_logger.debug('Tracked regex refreshed')
             print('Tracked regex refreshed')
 
-        item = Item(item_id)
+        obj = ail_objects.get_object(obj_type, subtype, obj_id)
-        item_id = item.get_id()
+        obj_id = obj.get_id()
-        if not content:
+        obj_type = obj.get_type()
-            content = item.get_content()
 
-        for regex in self.dict_regex_tracked:
+        # Object Filter
-            matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content)
+        if obj_type not in self.tracked_regexs:
+            return None
+
+        content = obj.get_content(r_str=True)
+
+        for dict_regex in self.tracked_regexs[obj_type]:
+            matched = self.regex_findall(dict_regex['regex'], obj_id, content)
             if matched:
-                self.new_tracker_found(regex, 'regex', item)
+                self.new_tracker_found(dict_regex['tracked'], 'regex', obj)
 
-    def new_tracker_found(self, tracker_name, tracker_type, item):
+    def new_tracker_found(self, tracker_name, tracker_type, obj):
-        uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
+        obj_id = obj.get_id()
-
+        for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
-        item_id = item.get_id()
-        # date = item.get_date()
-        item_source = item.get_source()
-
-        for tracker_uuid in uuid_list:
             tracker = Tracker.Tracker(tracker_uuid)
 
-            # Source Filtering
+            # Filter Object
-            tracker_sources = tracker.get_sources()
+            filters = tracker.get_filters()
-            if tracker_sources and item_source not in tracker_sources:
+            if ail_objects.is_filtered(obj, filters):
                 continue
 
-            print(f'new tracked regex found: {tracker_name} in {item_id}')
+            print(f'new tracked regex found: {tracker_name} in {obj_id}')
-            self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
+            self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}')
-            # TODO
+
-            Tracker.add_tracked_item(tracker_uuid, item_id)
+            if obj.get_type() == 'item':
+                date = obj.get_date()
+            else:
+                date = None
+
+            tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
 
             for tag in tracker.get_tags():
-                msg = f'{tag};{item_id}'
+                if obj.get_type() == 'item':
+                    msg = f'{tag};{obj_id}'
                     self.add_message_to_queue(msg, 'Tags')
+                else:
+                    obj.add_tag(tag)
 
             if tracker.mail_export():
                 # TODO add matches + custom subjects
-                self.exporters['mail'].export(tracker, item)
+                self.exporters['mail'].export(tracker, obj)
 
             if tracker.webhook_export():
-                self.exporters['webhook'].export(tracker, item)
+                self.exporters['webhook'].export(tracker, obj)
 
 
 if __name__ == "__main__":
     module = Tracker_Regex()
     module.run()
+    # module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

bin/trackers/Tracker_Term.py

@@ -21,8 +21,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.Items import Item
+from lib.objects import ail_objects
-from packages import Term
 from lib import Tracker
 
 from exporter.MailExporter import MailExporterTracker
@@ -54,9 +53,9 @@ class Tracker_Term(AbstractModule):
         self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time")
 
         # loads tracked words
-        self.list_tracked_words = Term.get_tracked_words_list()
+        self.tracked_words = Tracker.get_tracked_words()
         self.last_refresh_word = time.time()
-        self.set_tracked_words_list = Term.get_set_tracked_words_list()
+        self.tracked_sets = Tracker.get_tracked_sets()
         self.last_refresh_set = time.time()
 
         # Exporter
@@ -65,93 +64,94 @@ class Tracker_Term(AbstractModule):
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
-    def compute(self, item_id, item_content=None):
+    def compute(self, obj_id, obj_type='item', subtype=''):
         # refresh Tracked term
-        if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
+        if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
-            self.list_tracked_words = Term.get_tracked_words_list()
+            self.tracked_words = Tracker.get_tracked_words()
             self.last_refresh_word = time.time()
             self.redis_logger.debug('Tracked word refreshed')
             print('Tracked word refreshed')
 
-        if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
+        if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
-            self.set_tracked_words_list = Term.get_set_tracked_words_list()
+            self.tracked_sets = Tracker.get_tracked_sets()
             self.last_refresh_set = time.time()
             self.redis_logger.debug('Tracked set refreshed')
             print('Tracked set refreshed')
 
-        # Cast message as Item
+        obj = ail_objects.get_object(obj_type, subtype, obj_id)
-        item = Item(item_id)
+        obj_type = obj.get_type()
-        if not item_content:
+
-            item_content = item.get_content()
+        # Object Filter
+        if obj_type not in self.tracked_words and obj_type not in self.tracked_sets:
+            return None
+
+        content = obj.get_content(r_str=True)
 
         signal.alarm(self.max_execution_time)
 
         dict_words_freq = None
         try:
-            dict_words_freq = Term.get_text_word_frequency(item_content)
+            dict_words_freq = Tracker.get_text_word_frequency(content)
         except TimeoutException:
-            self.redis_logger.warning(f"{item.get_id()} processing timeout")
+            self.redis_logger.warning(f"{obj.get_id()} processing timeout")
         else:
             signal.alarm(0)
 
         if dict_words_freq:
-            # create token statistics
-            # for word in dict_words_freq:
-            #    Term.create_token_statistics(item_date, word, dict_words_freq[word])
 
             # check solo words
-            # ###### # TODO: check if source needed #######
+            for word in self.tracked_words[obj_type]:
-            for word in self.list_tracked_words:
                 if word in dict_words_freq:
-                    self.new_term_found(word, 'word', item)
+                    self.new_tracker_found(word, 'word', obj)
 
             # check words set
-            for elem in self.set_tracked_words_list:
+            for tracked_set in self.tracked_sets[obj_type]:
-                list_words = elem[0]
-                nb_words_threshold = elem[1]
-                word_set = elem[2]
                 nb_uniq_word = 0
-
+                for word in tracked_set['words']:
-                for word in list_words:
                     if word in dict_words_freq:
                         nb_uniq_word += 1
-                if nb_uniq_word >= nb_words_threshold:
+                if nb_uniq_word >= tracked_set['nb']:
-                    self.new_term_found(word_set, 'set', item)
+                    self.new_tracker_found(tracked_set['tracked'], 'set', obj)
 
-    def new_term_found(self, tracker_name, tracker_type, item):
+    def new_tracker_found(self, tracker_name, tracker_type, obj):  # TODO FILTER
-        uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
+        obj_id = obj.get_id()
 
-        item_id = item.get_id()
+        for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
-        item_source = item.get_source()
-
-        for tracker_uuid in uuid_list:
             tracker = Tracker.Tracker(tracker_uuid)
 
-            # Source Filtering
+            # Filter Object
-            tracker_sources = tracker.get_sources()
+            filters = tracker.get_filters()
-            if tracker_sources and item_source not in tracker_sources:
+            if ail_objects.is_filtered(obj, filters):
                 continue
 
-            print(f'new tracked term found: {tracker_name} in {item_id}')
+            print(f'new tracked term found: {tracker_name} in {obj_id}')
-            self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}')
+            self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}')
-            # TODO
+
-            Tracker.add_tracked_item(tracker_uuid, item_id)
+            if obj.get_type() == 'item':
+                date = obj.get_date()
+            else:
+                date = None
+            tracker.add(obj.get_type(), obj.get_subtype(), obj_id, date=date)
 
             # Tags
             for tag in tracker.get_tags():
-                msg = f'{tag};{item_id}'
+                if obj.get_type() == 'item':
+                    msg = f'{tag};{obj_id}'
                     self.add_message_to_queue(msg, 'Tags')
+                else:
+                    obj.add_tag(tag)
 
             # Mail
             if tracker.mail_export():
                 # TODO add matches + custom subjects
-                self.exporters['mail'].export(tracker, item)
+                self.exporters['mail'].export(tracker, obj)
 
             # Webhook
             if tracker.webhook_export():
-                self.exporters['webhook'].export(tracker, item)
+                self.exporters['webhook'].export(tracker, obj)
 
 
 if __name__ == '__main__':
     module = Tracker_Term()
     module.run()
+    # module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

bin/trackers/Tracker_Typo_Squatting.py

@@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
+from lib.objects import ail_objects
 from lib import Tracker
 
 from exporter.MailExporter import MailExporterTracker
@@ -36,8 +36,8 @@ class Tracker_Typo_Squatting(AbstractModule):
         self.pending_seconds = 5
 
         # Refresh typo squatting
-        self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
+        self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
-        self.last_refresh_typosquat = time.time()
+        self.last_refresh_typosquatting = time.time()
 
         # Exporter
         self.exporters = {'mail': MailExporterTracker(),
@@ -45,47 +45,59 @@ class Tracker_Typo_Squatting(AbstractModule):
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
-    def compute(self, message):
+    def compute(self, message, obj_type='item', subtype=''):
         # refresh Tracked typo
-        if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'):
+        if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
-            self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
+            self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
-            self.last_refresh_typosquat = time.time()
+            self.last_refresh_typosquatting = time.time()
             self.redis_logger.debug('Tracked typosquatting refreshed')
             print('Tracked typosquatting refreshed')
 
-        host, item_id = message.split()
+        host, obj_id = message.split()
+        obj = ail_objects.get_object(obj_type, subtype, obj_id)
+        obj_type = obj.get_type()
 
-        # Cast message as Item
+        # Object Filter
-        for tracker in self.typosquat_tracked_words_list:
+        if obj_type not in self.tracked_typosquattings:
-            if host in self.typosquat_tracked_words_list[tracker]:
+            return None
-                item = Item(item_id)
-                self.new_tracker_found(tracker, 'typosquatting', item)
 
-    def new_tracker_found(self, tracker, tracker_type, item):
+        for typo in self.tracked_typosquattings[obj_type]:
-        item_id = item.get_id()
+            if host in typo['domains']:
-        item_source = item.get_source()
+                self.new_tracker_found(typo['tracked'], 'typosquatting', obj)
-        print(f'new tracked typosquatting found: {tracker} in {item_id}')
-        self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
 
-        for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type):
+    def new_tracker_found(self, tracked, tracker_type, obj):
+        obj_id = obj.get_id()
+        for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracked):
             tracker = Tracker.Tracker(tracker_uuid)
 
-            # Source Filtering
+            # Filter Object
-            tracker_sources = tracker.get_sources()
+            filters = tracker.get_filters()
-            if tracker_sources and item_source not in tracker_sources:
+            if ail_objects.is_filtered(obj, filters):
                 continue
 
-            Tracker.add_tracked_item(tracker_uuid, item_id)
+            print(f'new tracked typosquatting found: {tracked} in {obj_id}')
+            self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}')
 
+            if obj.get_type() == 'item':
+                date = obj.get_date()
+            else:
+                date = None
+
+            tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
+
+            # Tags
             for tag in tracker.get_tags():
-                msg = f'{tag};{item_id}'
+                if obj.get_type() == 'item':
+                    msg = f'{tag};{obj_id}'
                     self.add_message_to_queue(msg, 'Tags')
+                else:
+                    obj.add_tag(tag)
 
             if tracker.mail_export():
-                self.exporters['mail'].export(tracker, item)
+                self.exporters['mail'].export(tracker, obj)
 
             if tracker.webhook_export():
-                self.exporters['webhook'].export(tracker, item)
+                self.exporters['webhook'].export(tracker, obj)
 
 
 if __name__ == '__main__':

bin/trackers/Tracker_Yara.py

@@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
+from lib.objects import ail_objects
 from lib import Tracker
 
 from exporter.MailExporter import MailExporterTracker
@@ -35,10 +35,10 @@ class Tracker_Yara(AbstractModule):
         self.pending_seconds = 5
 
         # Load Yara rules
-        self.rules = Tracker.reload_yara_rules()
+        self.rules = Tracker.get_tracked_yara_rules()
         self.last_refresh = time.time()
 
-        self.item = None
+        self.obj = None
 
         # Exporter
         self.exporters = {'mail': MailExporterTracker(),
@@ -46,58 +46,67 @@ class Tracker_Yara(AbstractModule):
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
-    def compute(self, item_id, item_content=None):
+    def compute(self, obj_id, obj_type='item', subtype=''):
         # refresh YARA list
         if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
-            self.rules = Tracker.reload_yara_rules()
+            self.rules = Tracker.get_tracked_yara_rules()
             self.last_refresh = time.time()
             self.redis_logger.debug('Tracked set refreshed')
             print('Tracked set refreshed')
 
-        self.item = Item(item_id)
+        self.obj = ail_objects.get_object(obj_type, subtype, obj_id)
-        if not item_content:
+        obj_type = self.obj.get_type()
-            item_content = self.item.get_content()
+
+        # Object Filter
+        if obj_type not in self.rules:
+            return None
+
+        content = self.obj.get_content(r_str=True)
 
         try:
-            yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
+            yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
                                                     which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
             if yara_match:
-                self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
+                self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}')
-                print(f'{self.item.get_id()}: {yara_match}')
+                print(f'{self.obj.get_id()}: {yara_match}')
         except yara.TimeoutError:
-            print(f'{self.item.get_id()}: yara scanning timed out')
+            print(f'{self.obj.get_id()}: yara scanning timed out')
-            self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
+            self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
 
     def yara_rules_match(self, data):
-        tracker_uuid = data['namespace']
+        tracker_name = data['namespace']
-        item_id = self.item.get_id()
+        obj_id = self.obj.get_id()
-
+        for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name):
-        item = Item(item_id)
-        item_source = self.item.get_source()
-
             tracker = Tracker.Tracker(tracker_uuid)
 
-        # Source Filtering
+            # Filter Object
-        tracker_sources = tracker.get_sources()
+            filters = tracker.get_filters()
-        if tracker_sources and item_source not in tracker_sources:
+            if ail_objects.is_filtered(self.obj, filters):
-            print(f'Source Filtering: {data["rule"]}')
+                continue
-            return yara.CALLBACK_CONTINUE
 
-        Tracker.add_tracked_item(tracker_uuid, item_id)  # TODO
+            if self.obj.get_type() == 'item':
+                date = self.obj.get_date()
+            else:
+                date = None
+
+            tracker.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id, date=date)
 
             # Tags
             for tag in tracker.get_tags():
-            msg = f'{tag};{item_id}'
+                if self.obj.get_type() == 'item':
+                    msg = f'{tag};{obj_id}'
                     self.add_message_to_queue(msg, 'Tags')
+                else:
+                    self.obj.add_tag(tag)
 
             # Mails
             if tracker.mail_export():
                 # TODO add matches + custom subjects
-            self.exporters['mail'].export(tracker, item)
+                self.exporters['mail'].export(tracker, self.obj)
 
             # Webhook
             if tracker.webhook_export():
-            self.exporters['webhook'].export(tracker, item)
+                self.exporters['webhook'].export(tracker, self.obj)
 
         return yara.CALLBACK_CONTINUE
 
@@ -105,3 +114,4 @@ class Tracker_Yara(AbstractModule):
 if __name__ == '__main__':
     module = Tracker_Yara()
     module.run()
+    # module.compute('archive/gist.github.com/2023/04/13/chipzoller_d8d6d2d737d02ad4fe9d30a897170761.gz')

update/v2.2/Update.py

@@ -39,74 +39,75 @@ if __name__ == '__main__':
 
     r_serv_term_stats.flushdb()
 
-    # convert all regex:
+    # Disabled. Checkout the v2.2 branch if you need it
-    all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
+    # # convert all regex:
-    for regex in all_regex:
+    # all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
-        tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
+    # for regex in all_regex:
-        mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
+    #     tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
-
+    #     mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
-        new_term = regex[1:-1]
+    #
-        res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
+    #     new_term = regex[1:-1]
-                                          'admin@admin.test')
+    #     res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
-        if res[1] == 200:
+    #                                       'admin@admin.test')
-            term_uuid = res[0]['uuid']
+    #     if res[1] == 200:
-            list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
+    #         term_uuid = res[0]['uuid']
-            for paste_item in list_items:
+    #         list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
-                item_id = get_item_id(paste_item)
+    #         for paste_item in list_items:
-                item_date = get_item_date(item_id)
+    #             item_id = get_item_id(paste_item)
-                Term.add_tracked_item(term_uuid, item_id, item_date)
+    #             item_date = get_item_date(item_id)
-
+    #             Term.add_tracked_item(term_uuid, item_id, item_date)
-        # Invalid Tracker => remove it
+    #
-        else:
+    #     # Invalid Tracker => remove it
-            print('Invalid Regex Removed: {}'.format(regex))
+    #     else:
-        print(res[0])
+    #         print('Invalid Regex Removed: {}'.format(regex))
-        # allow reprocess
+    #     print(res[0])
-        r_serv_termfreq.srem('TrackedRegexSet', regex)
+    #     # allow reprocess
-
+    #     r_serv_termfreq.srem('TrackedRegexSet', regex)
-    all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
+    #
-    for token in all_tokens:
+    # all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
-        tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
+    # for token in all_tokens:
-        mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
+    #     tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
-
+    #     mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
-        res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
+    #
-        if res[1] == 200:
+    #     res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
-            term_uuid = res[0]['uuid']
+    #     if res[1] == 200:
-            list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
+    #         term_uuid = res[0]['uuid']
-            for paste_item in list_items:
+    #         list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
-                item_id = get_item_id(paste_item)
+    #         for paste_item in list_items:
-                item_date = get_item_date(item_id)
+    #             item_id = get_item_id(paste_item)
-                Term.add_tracked_item(term_uuid, item_id, item_date)
+    #             item_date = get_item_date(item_id)
-        # Invalid Tracker => remove it
+    #             Term.add_tracked_item(term_uuid, item_id, item_date)
-        else:
+    #     # Invalid Tracker => remove it
-            print('Invalid Token Removed: {}'.format(token))
+    #     else:
-        print(res[0])
+    #         print('Invalid Token Removed: {}'.format(token))
-        # allow reprocess
+    #     print(res[0])
-        r_serv_termfreq.srem('TrackedSetTermSet', token)
+    #     # allow reprocess
-
+    #     r_serv_termfreq.srem('TrackedSetTermSet', token)
-    all_set = r_serv_termfreq.smembers('TrackedSetSet')
+    #
-    for curr_set in all_set:
+    # all_set = r_serv_termfreq.smembers('TrackedSetSet')
-        tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
+    # for curr_set in all_set:
-        mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
+    #     tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
-
+    #     mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
-        to_remove = ',{}'.format(curr_set.split(',')[-1])
+    #
-        new_set = rreplace(curr_set, to_remove, '', 1)
+    #     to_remove = ',{}'.format(curr_set.split(',')[-1])
-        new_set = new_set[2:]
+    #     new_set = rreplace(curr_set, to_remove, '', 1)
-        new_set = new_set.replace(',', '')
+    #     new_set = new_set[2:]
-
+    #     new_set = new_set.replace(',', '')
-        res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
+    #
-        if res[1] == 200:
+    #     res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
-            term_uuid = res[0]['uuid']
+    #     if res[1] == 200:
-            list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
+    #         term_uuid = res[0]['uuid']
-            for paste_item in list_items:
+    #         list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
-                item_id = get_item_id(paste_item)
+    #         for paste_item in list_items:
-                item_date = get_item_date(item_id)
+    #             item_id = get_item_id(paste_item)
-                Term.add_tracked_item(term_uuid, item_id, item_date)
+    #             item_date = get_item_date(item_id)
-        # Invalid Tracker => remove it
+    #             Term.add_tracked_item(term_uuid, item_id, item_date)
-        else:
+    #     # Invalid Tracker => remove it
-            print('Invalid Set Removed: {}'.format(curr_set))
+    #     else:
-        print(res[0])
+    #         print('Invalid Set Removed: {}'.format(curr_set))
-        # allow reprocess
+    #     print(res[0])
-        r_serv_termfreq.srem('TrackedSetSet', curr_set)
+    #     # allow reprocess
+    #     r_serv_termfreq.srem('TrackedSetSet', curr_set)
 
     r_serv_termfreq.flushdb()
 

update/v3.7/Update.py

@@ -24,7 +24,7 @@ class Updater(AIL_Updater):
         print('Fixing Tracker_uuid list ...')
         Tracker.fix_all_tracker_uuid_list()
         nb = 0
-        for tracker_uuid in Tracker.get_all_tracker_uuid():
+        for tracker_uuid in Tracker.get_trackers():
             self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
             nb += 1
 

var/www/blueprints/hunters.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
+"""
     Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
-'''
+"""
 
 import os
 import sys
 import json
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
+from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required, current_user, login_user, logout_user
 
 sys.path.append('modules')
@@ -22,8 +22,10 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
+from lib import ail_core
 from lib import item_basic
 from lib import Tracker
+from lib import Tag
 
 
 bootstrap_label = Flask_config.bootstrap_label
@@ -34,7 +36,6 @@ hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ
 # ============ VARIABLES ============
 
 
-
 # ============ FUNCTIONS ============
 def api_validator(api_response):
     if api_response:
@@ -45,11 +46,203 @@ def create_json_response(data, status_code):
 
 # ============= ROUTES ==============
 
+##################
+#    TRACKERS    #
+##################
+
+@hunters.route('/trackers', methods=['GET'])
+@login_required
+@login_read_only
+def trackers_dashboard():
+    user_id = current_user.get_id()  # TODO
+    trackers = Tracker.get_trackers_dashboard()
+    stats = Tracker.get_trackers_stats(user_id)
+    return render_template("trackers_dashboard.html", trackers=trackers, stats=stats, bootstrap_label=bootstrap_label)
+
+@hunters.route("/trackers/all")
+@login_required
+@login_read_only
+def tracked_menu():
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id)
+    global_trackers = Tracker.get_global_trackers_meta()
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
+
+@hunters.route("/trackers/word")
+@login_required
+@login_read_only
+def tracked_menu_word():
+    tracker_type = 'word'
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type='word')
+    global_trackers = Tracker.get_global_trackers_meta(tracker_type='word')
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
+
+@hunters.route("/trackers/set")
+@login_required
+@login_read_only
+def tracked_menu_set():
+    tracker_type = 'set'
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
+    global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
+
+@hunters.route("/trackers/regex")
+@login_required
+@login_read_only
+def tracked_menu_regex():
+    tracker_type = 'regex'
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
+    global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
+
+@hunters.route("/trackers/yara")
+@login_required
+@login_read_only
+def tracked_menu_yara():
+    tracker_type = 'yara'
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
+    global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
+
+@hunters.route("/trackers/typosquatting")
+@login_required
+@login_read_only
+def tracked_menu_typosquatting():
+    tracker_type = 'typosquatting'
+    user_id = current_user.get_id()
+    user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
+    global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
+    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
+                           bootstrap_label=bootstrap_label, tracker_type=tracker_type)
+
+@hunters.route("/tracker/add", methods=['GET', 'POST'])
+@login_required
+@login_analyst
+def add_tracked_menu():
+    if request.method == 'POST':
+        to_track = request.form.get("tracker")
+        tracker_uuid = request.form.get("tracker_uuid")
+        tracker_type = request.form.get("tracker_type")
+        nb_words = request.form.get("nb_word", 1)
+        description = request.form.get("description", '')
+        webhook = request.form.get("webhook", '')
+        level = request.form.get("level", 0)
+        mails = request.form.get("mails", [])
+
+        # TAGS
+        tags = request.form.get("tags", [])
+        taxonomies_tags = request.form.get('taxonomies_tags')
+        if taxonomies_tags:
+            try:
+                taxonomies_tags = json.loads(taxonomies_tags)
+            except:
+                taxonomies_tags = []
+        else:
+            taxonomies_tags = []
+        galaxies_tags = request.form.get('galaxies_tags')
+        if galaxies_tags:
+            try:
+                galaxies_tags = json.loads(galaxies_tags)
+            except:
+                galaxies_tags = []
+        # custom tags
+        if tags:
+            tags = tags.split()
+        else:
+            tags = []
+        tags = tags + taxonomies_tags + galaxies_tags
+
+        # YARA #
+        if tracker_type == 'yara':
+            yara_default_rule = request.form.get("yara_default_rule")
+            yara_custom_rule = request.form.get("yara_custom_rule")
+            if yara_custom_rule:
+                to_track = yara_custom_rule
+                tracker_type = 'yara_custom'
+            else:
+                to_track = yara_default_rule
+                tracker_type = 'yara_default'
+
+        if level == 'on':
+            level = 1
+        else:
+            level = 0
+        if mails:
+            mails = mails.split()
+        else:
+            tags = []
+            
+        # FILTERS
+        filters = {}
+        for obj_type in Tracker.get_objects_tracked():
+            new_filter = request.form.get(f'{obj_type}_obj')
+            if new_filter == 'on':
+                filters[obj_type] = {}
+                # Mimetypes
+                mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
+                if mimetypes:
+                    mimetypes = json.loads(mimetypes)
+                    filters[obj_type]['mimetypes'] = mimetypes
+                # Sources
+                sources = request.form.get(f'sources_{obj_type}', [])
+                if sources:
+                    sources = json.loads(sources)
+                    filters[obj_type]['sources'] = sources
+                # Subtypes
+                for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
+                    subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
+                    if subtype == 'on':
+                        if 'subtypes' not in filters[obj_type]:
+                            filters[obj_type]['subtypes'] = []
+                        filters[obj_type]['subtypes'].append(obj_subtype)
+
+        input_dict = {"tracked": to_track, "type": tracker_type,
+                      "tags": tags, "mails": mails, "filters": filters,
+                      "level": level, "description": description, "webhook": webhook}
+        if tracker_type == 'set':
+            try:
+                input_dict['nb_words'] = int(nb_words)
+            except TypeError:
+                input_dict['nb_words'] = 1
+
+        user_id = current_user.get_id()
+        res = Tracker.api_add_tracker(input_dict, user_id)
+        if res[1] == 200:
+            return redirect(url_for('hunters.trackers_dashboard'))
+        else:
+            return create_json_response(res[0], res[1])
+    else:
+        return render_template("tracker_add.html",
+                                all_sources=item_basic.get_all_items_sources(r_list=True),
+                                tags_selector_data=Tag.get_tags_selector_data(),
+                                all_yara_files=Tracker.get_all_default_yara_files())
+
+@hunters.route('/tracker/delete', methods=['GET'])
+@login_required
+@login_analyst
+def tracker_delete():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid')
+    res = Tracker.api_delete_tracker({'uuid': tracker_uuid}, user_id)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        return redirect(url_for('hunter.tracked_menu'))
+
+
+####################
+#    RETRO HUNT    #
+####################
+
 @hunters.route('/retro_hunt/tasks', methods=['GET'])
 @login_required
 @login_read_only
 def retro_hunt_all_tasks():
-    retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata()
+    retro_hunts = Tracker.get_retro_hunt_tasks_metas()
     return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
 
 @hunters.route('/retro_hunt/task/show', methods=['GET'])
@@ -69,8 +262,8 @@ def retro_hunt_show_task():
     if res:
         return create_json_response(res[0], res[1])
 
-    dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True,
+    retro_hunt = Tracker.RetroHunt(task_uuid)
-                                                        sources=True, tags=True, description=True)
+    dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
     rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
 
     if date_from:
@@ -177,7 +370,7 @@ def retro_hunt_delete_task():
 
 #### JSON ####
 
-@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET'])
+@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
 @login_required
 @login_read_only
 def get_json_retro_hunt_nb_items_by_date():

var/www/modules/hunter/Flask_hunter.py

@@ -17,10 +17,11 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
+from lib.objects import ail_objects
 from lib import item_basic
 from lib import Tracker
 from lib import Tag
-from packages import Term
+from packages import Date
 
 
 # ============ VARIABLES ============
@@ -34,142 +35,11 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
 
 # ============ FUNCTIONS ============
 
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
 # ============ ROUTES ============
 
-@hunter.route("/trackers")
-@login_required
-@login_read_only
-def tracked_menu():
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id)
-    global_trackers = Tracker.get_global_trackers_metadata()
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
-
-@hunter.route("/trackers/word")
-@login_required
-@login_read_only
-def tracked_menu_word():
-    tracker_type = 'word'
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
-    global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
-
-@hunter.route("/trackers/set")
-@login_required
-@login_read_only
-def tracked_menu_set():
-    tracker_type = 'set'
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
-    global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
-
-@hunter.route("/trackers/regex")
-@login_required
-@login_read_only
-def tracked_menu_regex():
-    tracker_type = 'regex'
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
-    global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
-
-@hunter.route("/trackers/yara")
-@login_required
-@login_read_only
-def tracked_menu_yara():
-    tracker_type = 'yara'
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
-    global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
-
-@hunter.route("/trackers/typosquatting")
-@login_required
-@login_read_only
-def tracked_menu_typosquatting():
-    tracker_type = 'typosquatting'
-    user_id = current_user.get_id()
-    user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
-    global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
-    return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
-                           bootstrap_label=bootstrap_label, tracker_type=tracker_type)
-
-@hunter.route("/tracker/add", methods=['GET', 'POST'])
-@login_required
-@login_analyst
-def add_tracked_menu():
-    if request.method == 'POST':
-        tracker = request.form.get("tracker")
-        tracker_uuid = request.form.get("tracker_uuid")
-        tracker_type  = request.form.get("tracker_type")
-        nb_words = request.form.get("nb_word", 1)
-        description = request.form.get("description", '')
-        webhook = request.form.get("webhook", '')
-        level = request.form.get("level", 0)
-        mails = request.form.get("mails", [])
-        sources = request.form.get("sources", [])
-
-        tags = request.form.get("tags", [])
-        taxonomies_tags = request.form.get('taxonomies_tags')
-        if taxonomies_tags:
-            try:
-                taxonomies_tags = json.loads(taxonomies_tags)
-            except Exception:
-                taxonomies_tags = []
-        else:
-            taxonomies_tags = []
-        galaxies_tags = request.form.get('galaxies_tags')
-        if galaxies_tags:
-            try:
-                galaxies_tags = json.loads(galaxies_tags)
-            except Exception:
-                galaxies_tags = []
-
-
-        # YARA #
-        if tracker_type == 'yara':
-            yara_default_rule = request.form.get("yara_default_rule")
-            yara_custom_rule =  request.form.get("yara_custom_rule")
-            if yara_custom_rule:
-                tracker = yara_custom_rule
-                tracker_type='yara_custom'
-            else:
-                tracker = yara_default_rule
-                tracker_type='yara_default'
-
-        if level == 'on':
-            level = 1
-        if mails:
-            mails = mails.split()
-        if tags:
-            tags = tags.split()
-        if sources:
-            sources = json.loads(sources)
-
-        input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words,
-                        "tags": tags, "mails": mails, "sources": sources,
-                        "level": level, "description": description, "webhook": webhook}
-        user_id = current_user.get_id()
-        # edit tracker
-        if tracker_uuid:
-            input_dict['uuid'] = tracker_uuid
-        res = Tracker.api_add_tracker(input_dict, user_id)
-        if res[1] == 200:
-            if 'uuid' in res[0]:
-                return redirect(url_for('hunter.show_tracker', uuid=res[0]['uuid']))
-            else:
-                return redirect(url_for('hunter.tracked_menu'))
-        else:
-            ## TODO: use modal
-            return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
-    else:
-        return render_template("edit_tracker.html",
-                                all_sources=item_basic.get_all_items_sources(r_list=True),
-                                tags_selector_data=Tag.get_tags_selector_data(),
-                                all_yara_files=Tracker.get_all_default_yara_files())
-
 @hunter.route("/tracker/edit", methods=['GET', 'POST'])
 @login_required
 @login_analyst
@@ -181,7 +51,8 @@ def edit_tracked_menu():
     if res[1] != 200: # invalid access
         return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
-    dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True)
+    tracker = Tracker.Tracker(tracker_uuid)
+    dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'sources', 'tags', 'user', 'webhook'})
     dict_tracker['tags'] = ' '.join(dict_tracker['tags'])
     dict_tracker['mails'] = ' '.join(dict_tracker['mails'])
 
@@ -205,7 +76,7 @@ def edit_tracked_menu():
     # word
     # set of word + nb words
     # regex
-    # yara custum
+    # yara custom
     # yara default ???? => allow edit ?
 
     #### EDIT SHow Trackers ??????????????????????????????????????????????????
@@ -228,34 +99,36 @@ def show_tracker():
     if date_to:
         date_to = date_to.replace('-', '')
 
-    tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True)
+    tracker = Tracker.Tracker(tracker_uuid)
+    meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
+                                     'user', 'webhook'})
 
-    if tracker_metadata['type'] == 'yara':
+    if meta['type'] == 'yara':
-        yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
+        yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
     else:
         yara_rule_content = None
    
-    if tracker_metadata['type'] == 'typosquatting':
+    if meta['type'] == 'typosquatting':
-        typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
+        typo_squatting = Tracker.get_tracked_typosquatting_domains(meta['tracked'])
-        typo_squatting.sort()
+        sorted(typo_squatting)
     else:
-        typo_squatting = None
+        typo_squatting = set()
 
     if date_from:
-        res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
+        date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen'])
-        if res[1] != 200:
+        objs = tracker.get_objs_by_daterange(date_from, date_to)
-            return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+        meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True)
-        tracker_metadata['items'] = res[0]['items']
-        tracker_metadata['date_from'] = res[0]['date_from']
-        tracker_metadata['date_to'] = res[0]['date_to']
     else:
-        tracker_metadata['items'] = []
+        date_from = ''
-        tracker_metadata['date_from'] = ''
+        date_to = ''
-        tracker_metadata['date_to'] = ''
+        meta['objs'] = []
 
-    tracker_metadata['sources'] = sorted(tracker_metadata['sources'])
+    meta['date_from'] = date_from
+    meta['date_to'] = date_to
+    print(meta['filters'])
+    meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
 
-    return render_template("showTracker.html", tracker_metadata=tracker_metadata,
+    return render_template("showTracker.html", tracker_metadata=meta,
                            yara_rule_content=yara_rule_content,
                            typo_squatting=typo_squatting,
                            bootstrap_label=bootstrap_label)
@@ -309,21 +182,16 @@ def update_tracker_mails():
         return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
     return redirect(url_for('hunter.show_tracker', uuid=term_uuid))
 
-@hunter.route("/tracker/delete", methods=['GET'])
-@login_required
-@login_analyst
-def delete_tracker():
-    user_id = current_user.get_id()
-    term_uuid = request.args.get('uuid')
-    res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id)
-    if res[1] !=200:
-        return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
-    return redirect(url_for('hunter.tracked_menu'))
-
 @hunter.route("/tracker/get_json_tracker_stats", methods=['GET'])
 @login_required
 @login_read_only
 def get_json_tracker_stats():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid')
+    res = Tracker.api_check_tracker_acl(tracker_uuid, user_id)
+    if res:
+        return create_json_response(res[0], res[1])
+
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
 
@@ -331,13 +199,10 @@ def get_json_tracker_stats():
         date_from = date_from.replace('-', '')
     if date_to:
         date_to = date_to.replace('-', '')
-
-    tracker_uuid = request.args.get('uuid')
-
     if date_from and date_to:
-        res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
+        res = Tracker.get_trackers_graph_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
     else:
-        res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
+        res = Tracker.get_trackers_graph_by_day([tracker_uuid])
     return jsonify(res)
 
 @hunter.route("/tracker/yara/default_rule/content", methods=['GET'])
@@ -348,5 +213,6 @@ def get_default_yara_rule_content():
     res = Tracker.api_get_default_rule_content(default_yara_rule)
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
+
 # ========= REGISTRATION =========
 app.register_blueprint(hunter, url_prefix=baseUrl)

var/www/modules/hunter/templates/edit_tracker.html

@@ -1,260 +0,0 @@
-<!DOCTYPE html>
-
-<html>
-<head>
-	<title>AIL-Framework</title>
-	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
-	<!-- Core CSS -->
-	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
-
-	<!-- JS -->
-	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
-
-</head>
-
-<body>
-
-	{% include 'nav_bar.html' %}
-
-	<div class="container-fluid">
-		<div class="row">
-
-			{% include 'hunter/menu_sidebar.html' %}
-
-			<div class="col-12 col-lg-10" id="core_content">
-
-						<div class="card my-3">
-						  <div class="card-header bg-dark text-white">
-								<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
-							</div>
-						  <div class="card-body">
-
-								<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
-									{%if dict_tracker%}
-										<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
-									{%endif%}
-
-									<div class="row">
-										<div class="col-12 col-xl-9">
-											<div class="input-group mb-2 mr-sm-2">
-										    <div class="input-group-prepend">
-										      <div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
-										    </div>
-												<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
-										  </div>
-											<div class="input-group mb-2 mr-sm-2">
-										    <div class="input-group-prepend">
-										      <div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
-										    </div>
-												<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
-										  </div>
-											<div class="input-group mb-2 mr-sm-2">
-										    <div class="input-group-prepend">
-										      <div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
-										    </div>
-												<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
-										  </div>
-
-											<div class="input-group mb-2 mr-sm-2">
-												<div class="input-group-prepend">
-										      <div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
-										    </div>
-												<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
-											</div>
-
-
-											<div class="card my-4">
-											  <div class="card-header bg-secondary text-white">
-													<b>Tags</b>
-												</div>
-											  <div class="card-body">
-													<div class="input-group mb-2 mr-sm-2">
-														<div class="input-group-prepend">
-													  	<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
-													  </div>
-														<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
-													</div>
-													{% include 'tags/block_tags_selector.html' %}
-												</div>
-											</div>
-
-										</div>
-										<div class="col-12 col-xl-3">
-											<div class="custom-control custom-switch mt-1">
-												<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
-												<label class="custom-control-label" for="id_level">
-													<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
-												</label>
-											</div>
-										</div>
-									</div>
-
-									<hr>
-									<h4>Tracker Type:</h4>
-
-									<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
-										<option disabled selected value> -- Select a tracker type -- </option>
-										<option value="word">Word</option>
-										<option value="set">Set</option>
-										<option value="regex">Regex</option>
-										<option value="yara">YARA rule</option>
-										<option value="typosquatting">Typo-squatting</option>
-									</select>
-
-									<p id="tracker_desc">Terms to track (space separated)</p>
-
-									<div class="row" id="simple_input">
-										<div class="col-12 col-lg-10">
-											<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
-										</div>
-										<div class="col-12 col-lg-2">
-											<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
-										</div>
-									</div>
-
-
-									<div class="mb-2" id="yara_rule">
-										<div class="" id="yara_default_rules">
-
-											<h6>Default YARA rules:</h6>
-											<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
-											  <option selected>Select a default rule</option>
-												{% for yara_types in all_yara_files %}
-													{% for yara_file_name in all_yara_files[yara_types] %}
-													<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
-													{% endfor %}
-												{% endfor %}
-											</select>
-
-											<pre class="border bg-light" id="default_yara_rule_content"></pre>
-
-										</div>
-
-										<hr>
-
-										<h6>Custom YARA rules:</h6>
-										<div class="row" id="textarea">
-											<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
-										</div>
-									</div>
-
-									<br>
-									<button class="btn btn-success mt-2">
-										<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
-									</button>
-
-								</form>
-
-
-
-						  </div>
-						</div>
-
-
-			</div>
-
-		</div>
-	</div>
-
-</body>
-
-<script>
-var ltags;
-var chart = {};
-$(document).ready(function(){
-	$("#page-Tracker").addClass("active");
-	$("#nav_manual_crawler").addClass("active");
-	$("#tracker_desc").hide();
-	$("#tracker").hide();
-	$("#nb_word").hide();
-	$("#yara_rule").hide();
-
-	sources = $('#sources').tagSuggest({
-		data: {{all_sources|safe}},
-		{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
-		sortOrder: 'name',
-		maxDropHeight: 200,
-		name: 'sources',
-		emptyText: 'Sources to track (ALL IF EMPTY)',
-		});
-
-	$('#tracker_type').on('change', function() {
-		var tracker_type = this.value;
-		if (tracker_type=="word") {
-			$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
-			$("#tracker_desc").show();
-			$("#tracker").show();
-			$("#nb_word").hide();
-			$("#yara_rule").hide();
-		} else if (tracker_type=="set") {
-			$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
-			$("#tracker_desc").show();
-			$("#tracker").show();
-			$("#nb_word").show();
-			$("#yara_rule").hide();
-		} else if (tracker_type=="regex") {
-			$("#tracker_desc").text("Enter a valid Python regex");
-			$("#tracker_desc").show();
-			$("#tracker").show();
-			$("#nb_word").hide();
-			$("#yara_rule").hide();
-		} else if (tracker_type=="yara") {
-			$("#tracker_desc").text("Select a default yara rule or create your own rule:");
-			$("#tracker_desc").show();
-			$("#tracker").hide();
-			$("#nb_word").hide();
-			$("#yara_rule").show();
-		} else if (tracker_type=="typosquatting") {
-			$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
-			$("#tracker_desc").show();
-			$("#tracker").show();
-			$("#nb_word").hide();
-			$("#yara_rule").hide();
-		}
-	});
-
-	{%if dict_tracker%}
-		$('#tracker_type').val('{{dict_tracker['type']}}').change();
-
-		{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
-			$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
-		{%endif%}
-	{%endif%}
-
-});
-
-function toggle_sidebar(){
-	if($('#nav_menu').is(':visible')){
-		$('#nav_menu').hide();
-		$('#side_menu').removeClass('border-right')
-		$('#side_menu').removeClass('col-lg-2')
-		$('#core_content').removeClass('col-lg-10')
-	}else{
-		$('#nav_menu').show();
-		$('#side_menu').addClass('border-right')
-		$('#side_menu').addClass('col-lg-2')
-		$('#core_content').addClass('col-lg-10')
-	}
-}
-
-
-
-function get_default_rule_content(selector){
-	var yara_name = selector.value
-	if (yara_name === "Select a default rule") {
-		jQuery("#default_yara_rule_content").text("")
-	} else {
-		$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
-			function(data) {
-				jQuery("#default_yara_rule_content").text(data['content'])
-		});
-	}
-}
-
-</script>

var/www/modules/hunter/templates/showTracker.html

@@ -72,7 +72,7 @@
                                             <tr>
                                                 <th>Type</th>
                                                 <th>Tracker</th>
-                                                <th>Date added</th>
+                                                <th>Created</th>
                                                 <th>Access Level</th>
                                                 <th>Created by</th>
                                                 <th>First seen</th>
@@ -94,7 +94,7 @@
                                             {% if tracker_metadata['type'] == 'typosquatting' %}
                                                 <td>
                                                     <a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
-                                                        {{ tracker_metadata['tracker'].split(",")[0] }}
+                                                        {{ tracker_metadata['tracked'].split(",")[0] }}
                                                     </a>
                                                     <div class="collapse" id="collapseTypo">
                                                         <div class="card card-body">
@@ -108,7 +108,7 @@
                                                     </div>
                                                 </td>
                                             {% else %}
-                                                <td>{{ tracker_metadata['tracker'] }}</td>
+                                                <td>{{ tracker_metadata['tracked'] }}</td>
                                             {% endif %}
                                             <td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
                                             <td>
@@ -118,19 +118,15 @@
                                                     Global
                                                 {% endif %}
                                             </td>
-                                            <td>{{ tracker_metadata['user_id'] }}</td>
+                                            <td>{{ tracker_metadata['user'] }}</td>
                                             <td>
                                                 {% if tracker_metadata['first_seen'] %}
-                                                    {{ tracker_metadata['first_seen'][0:4] }}/
+                                                    {{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }}
-                                                    {{ tracker_metadata['first_seen'][4:6] }}/
-                                                    {{ tracker_metadata['first_seen'][6:8] }}
                                                 {% endif %}
                                             </td>
                                             <td>
                                                 {% if tracker_metadata['last_seen'] %}
-                                                    {{ tracker_metadata['last_seen'][0:4] }}/
+                                                    {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }}
-                                                    {{ tracker_metadata['last_seen'][4:6] }}/
-                                                    {{ tracker_metadata['last_seen'][6:8] }}
                                                 {% endif %}
                                             </td>
                                             {% if tracker_metadata['webhook'] %}
@@ -159,14 +155,22 @@
                                     <div id="sparkline"></div>
                                 </div>
                             </div>
-                            <h6>Sources:</h6>
+
-                            {% if tracker_metadata['sources'] %}
+                            <h6>Filters:</h6>
-                                {% for sources in tracker_metadata['sources'] %}
+                            {% if tracker_metadata['filters'] %}
-                                    <span class="badge badge-secondary">{{ sources }}</span><br>
+                                <pre>{{ tracker_metadata['filters'] }}</pre>
-                                {% endfor %}
                             {% else %}
-                                <span class="badge badge-secondary">All Souces</span><br>
+                                <span class="badge badge-secondary">No Filters</span><br>
                             {% endif %}
+
+{#                            <h6>Sources:</h6>#}
+{#                            {% if tracker_metadata['sources'] %}#}
+{#                                {% for sources in tracker_metadata['sources'] %}#}
+{#                                    <span class="badge badge-secondary">{{ sources }}</span><br>#}
+{#                                {% endfor %}#}
+{#                            {% else %}#}
+{#                                <span class="badge badge-secondary">All Sources</span><br>#}
+{#                            {% endif %}#}
                         </li>
                     </ul>
 
@@ -232,7 +236,7 @@
                     </div>
 
                     <div class="d-flex flex-row-reverse">
-                        <a href="{{ url_for('hunter.delete_tracker') }}?uuid={{ tracker_metadata['uuid'] }}"
+                        <a href="{{ url_for('hunters.tracker_delete') }}?uuid={{ tracker_metadata['uuid'] }}"
                            style="font-size: 15px">
                             <button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
                         </a>
@@ -262,8 +266,9 @@
                                         class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
                                 <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd"
                                        name="date_from" autocomplete="off"
-                                       {% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'] }}"
+                                       {% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'][0:4] }}-{{ tracker_metadata['date_from'][4:6] }}-{{ tracker_metadata['date_from'][6:8] }}"
-                                       {% else %}value="{{ tracker_metadata['first_seen'] }}"{% endif %}>
+                                       {% elif tracker_metadata['first_seen'] %}value="{{ tracker_metadata['first_seen'][0:4] }}-{{ tracker_metadata['first_seen'][4:6] }}-{{ tracker_metadata['first_seen'][6:8] }}"
+                                       {% endif %}>
                             </div>
                         </div>
                         <div class="col-md-6">
@@ -272,8 +277,9 @@
                                         class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
                                 <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd"
                                        name="date_to" autocomplete="off"
-                                       {% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'] }}"
+                                       {% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'][0:4] }}-{{ tracker_metadata['date_to'][4:6] }}-{{ tracker_metadata['date_to'][6:8] }}"
-                                       {% else %}value="{{ tracker_metadata['last_seen'] }}"{% endif %}>
+                                       {% elif tracker_metadata['last_seen'] %}value="{{ tracker_metadata['last_seen'][0:4] }}-{{ tracker_metadata['last_seen'][4:6] }}-{{ tracker_metadata['last_seen'][6:8] }}"
+                                       {% endif %}>
                             </div>
                         </div>
                     </div>
@@ -285,40 +291,52 @@
                 </div>
             </div>
 
-            {% if tracker_metadata['items'] %}
+            {% if tracker_metadata['objs'] %}
                 <div class="mt-4">
-                    <table class="table table-bordered table-hover" id="myTable_">
+                    <table id="myTable_" class="table table-striped border-primary">
-                        <thead class="thead-dark">
+                        <thead class="bg-dark text-white">
                         <tr>
-                            <th>Date</th>
+                            <th>Type</th>
-                            <th>Item Id</th>
+			      	        <th></th>
+							<th>Id</th>
+				     	    <th>Tags</th>
+							<th></th>
 				   	    </tr>
 					    </thead>
-                        <tbody>
+					    <tbody style="font-size: 15px;">
-
+						{% for object in tracker_metadata['objs'] %}
-                        {% for item in tracker_metadata['items'] %}
+							<tr class="border-color: blue;">
-                            <tr>
 								<td>
-                                    {{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }}
+									{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
+										{% include 'objects/obj_svg_block.html' %}
+									{% endwith %}
+									{{ object['type']}}
 								</td>
 								<td>
-                                    <a class="text-secondary" target="_blank"
+									{% if object['subtype'] %}
-                                       href="{{ url_for('objects_item.showItem') }}?id={{ item['id'] }}">
+										{{ object['subtype']}}
-                                        <div style="line-height:0.9;">{{ item['id'] }}</div>
+									{% endif %}
+								</td>
+								<td>
+									<a href="{{ object['link'] }}">
+										{{ object['id']}}
 									</a>
-                                    <div class="mb-2">
+								</td>
-                                        {% for tag in item['tags'] %}
+								<td>
-                                            <a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
+									{% for tag in object['tags'] %}
 										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
-                                            </a>
 									{% endfor %}
-                                    </div>
+								</td>
+								<td class="text-right">
+{#									<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
+{#										<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
+{#									</a>#}
 								</td>
 							</tr>
 						{% endfor %}
-
 					    </tbody>
 				    </table>
+
                 </div>
             {% endif %}
 
@@ -331,7 +349,7 @@
         $('#div_edit_mails').hide();
         $('#div_edit_tags').hide();
         $('#div_edit_description').hide();
-        $("#page-Decoded").addClass("active");
+        $("#page-Tracker").addClass("active");
 
         $('#date-range-from').dateRangePicker({
             separator: ' to ',

var/www/modules/restApi/Flask_restApi.py

@@ -22,8 +22,6 @@ from lib.objects import Items
 from lib import Tag
 from lib import Tracker
 
-from packages import Term  # TODO REMOVE ME
-
 from packages import Import_helper
 
 from importer.FeederImporter import api_add_json_feeder_to_queue
@@ -324,9 +322,10 @@ def get_all_tags():
     res = {'tags': Tag.get_all_tags()}
     return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
 
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
-# # # # # # # # # # # # # #        TRACKER       # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # #        TRACKER      # # # # # # # # # # # # # # # # # TODO
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
+'''
 @restApi.route("api/v1/add/tracker", methods=['POST'])
 @token_required('analyst')
 def add_tracker_term():
@@ -371,11 +370,24 @@ def get_tracker_metadata_api():
     data = request.get_json()
     tracker_uuid = data.get('tracker_uuid', None)
     req_data = {'tracker_uuid': tracker_uuid}
+    
+    tracker_uuid = request_dict.get('tracker_uuid', None)
+    if not request_dict:
+        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+    if not tracker_uuid:
+        return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
+    if not is_valid_uuid_v4(tracker_uuid):
+        return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
+    if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
+        return {'status': 'error', 'reason': 'Tracker not found'}, 404
+        
     res = Tracker.get_tracker_metadata_api(req_data)
     return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
     
 '''
 
+'''
+
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # #        CRYPTOCURRENCY       # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -530,27 +542,27 @@ def get_domain_status_minimal():
     res[0]['domain'] = domain
     return create_json_response(res[0], res[1])
 
-@restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
+# @restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
-@token_required('analyst')
+# @token_required('analyst')
-def get_crawled_domain_list():
+# def get_crawled_domain_list():
-    data = request.get_json()
+#     data = request.get_json()
-    res = get_mandatory_fields(data, ['date_from', 'date_to'])
+#     res = get_mandatory_fields(data, ['date_from', 'date_to'])
-    if res:
+#     if res:
-        return create_json_response(res[0], res[1])
+#         return create_json_response(res[0], res[1])
-
+#
-    date_from = data.get('date_from', None)
+#     date_from = data.get('date_from', None)
-    date_to = data.get('date_to', None)
+#     date_to = data.get('date_to', None)
-    domain_type = data.get('domain_type', None)
+#     domain_type = data.get('domain_type', None)
-    domain_status = 'UP'
+#     domain_status = 'UP'
-    # TODO TO MIGRATE
+#     # TODO TO MIGRATE
-    raise Exception('TO MIGRATE')
+#     raise Exception('TO MIGRATE')
-    # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
+#     # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
-    dict_res = res[0]
+#     dict_res = res[0]
-    dict_res['date_from'] = date_from
+#     dict_res['date_from'] = date_from
-    dict_res['date_to'] = date_to
+#     dict_res['date_to'] = date_to
-    dict_res['domain_status'] = domain_status
+#     dict_res['domain_status'] = domain_status
-    dict_res['domain_type'] = domain_type
+#     dict_res['domain_type'] = domain_type
-    return create_json_response(dict_res, res[1])
+#     return create_json_response(dict_res, res[1])
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # #        IMPORT     # # # # # # # # # # # # # # # # # #

var/www/templates/hunter/menu_sidebar.html

@@ -8,43 +8,43 @@
       	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
       		<h5 class="d-flex text-muted w-100" id="nav_title_trackers">
       		  <span>Trackers </span>
-      			<a class="ml-auto" href="{{url_for('hunter.add_tracked_menu')}}">
+      			<a class="ml-auto" href="{{url_for('hunters.add_tracked_menu')}}">
       				<i class="fas fa-plus-circle ml-auto"></i>
       			</a>
       		</h5>
       	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
             <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu')}}" id="nav_tracker_">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu')}}" id="nav_tracker_">
                 <i class="fas fa-ruler-combined"></i>
       	        <span>All Trackers</span>
       	      </a>
       	    </li>
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu_word')}}" id="nav_tracker_word">
                 <i class="fas fa-font"></i>
       	        <span>Words</span>
       	      </a>
       	    </li>
             <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu_set')}}" id="nav_tracker_set">
       	        <i class="fas fa-layer-group"></i>
       	        <span>Set</span>
       	      </a>
       	    </li>
             <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu_regex')}}" id="nav_tracker_regex">
       	        <i class="fas fa-drafting-compass"></i>
                 <span>Regex</span>
       	      </a>
       	    </li>
             <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu_yara')}}" id="nav_tracker_yara">
                 <span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
                 <span>&nbsp;YARA</span>
       	      </a>
       	    </li>
 			<li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
+      	      <a class="nav-link" href="{{url_for('hunters.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
                 <i class="fa fa-clone"></i>
                 <span>Typo-squatting</span>
       	      </a>

var/www/templates/hunter/tracker_add.html

@@ -0,0 +1,396 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'hunter/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+						<div class="card my-3">
+						  <div class="card-header bg-dark text-white">
+								<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
+							</div>
+						  <div class="card-body">
+
+								<form action="{{ url_for('hunters.add_tracked_menu') }}" method='post'>
+									{%if dict_tracker%}
+										<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
+									{%endif%}
+
+									<div class="row">
+										<div class="col-12 col-xl-9">
+											<div class="input-group mb-2 mr-sm-2">
+										    <div class="input-group-prepend">
+                                                <div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
+										    </div>
+												<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
+										    </div>
+											<div class="input-group mb-2 mr-sm-2">
+										        <div class="input-group-prepend">
+                                                    <div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
+										        </div>
+												<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
+                                            </div>
+											<div class="input-group mb-2 mr-sm-2">
+                                                <div class="input-group-prepend">
+                                                    <div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
+                                                </div>
+												<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
+                                            </div>
+
+                                            <div class="card my-4">
+                                                <div class="card-header bg-info text-white">
+													<b>Objects to Track:</b>
+												</div>
+                                                <div class="card-body">
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i>&nbsp;CVE</label>#}
+{#                                                    </div>#}
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
+{#                                                    </div>#}
+                                                    <div class="custom-control custom-switch mt-1">
+                                                        <input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
+                                                        <label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
+                                                    </div>
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
+{#                                                    </div>#}
+                                                    <div class="custom-control custom-switch mt-1">
+                                                        <input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
+                                                        <label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item</label>
+                                                    </div>
+                                                    <div class="card border-dark mb-4" id="sources_item_div">
+                                                        <div class="card-body">
+                                                            <h5>Filter Item by sources</h5>
+                                                            <div class="input-group mb-2 mr-sm-2">
+                                                                <div class="input-group-prepend">
+                                                                    <div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
+                                                                </div>
+                                                                <input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
+                                                            </div>
+                                                        </div>
+                                                    </div>
+
+
+                                                    <div class="custom-control custom-switch mt-1">
+                                                        <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
+                                                        <label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP</label>
+                                                    </div>
+                                                    <div class="card border-dark mb-4" id="sources_pgp_div">
+                                                        <div class="card-body">
+                                                            <h6>Filter PGP by subtype:</h6>
+                                                            <div class="custom-control custom-switch mt-1">
+                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
+                                                                <label class="custom-control-label" for="filter_pgp_name">
+                                                                    <svg height="26" width="26">
+                                                                        <g class="nodes">
+                                                                            <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
+                                                                            <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
+                                                                        </g>
+                                                                    </svg>
+                                                                    name
+                                                                </label>
+                                                            </div>
+                                                            <div class="custom-control custom-switch mt-1">
+                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
+                                                                <label class="custom-control-label" for="filter_pgp_mail">
+                                                                    <svg height="26" width="26">
+                                                                        <g class="nodes">
+                                                                            <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
+                                                                            <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px">&#xf1fa;</text>
+                                                                        </g>
+                                                                        </svg>
+                                                                    mail
+                                                                </label>
+                                                            </div>
+                                                        </div>
+                                                    </div>
+
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="level" id="screenshot_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="screenshot_obj"><i class="fas fa-image"></i>&nbsp;Screenshot</label>#}
+{#                                                    </div>#}
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="level" id="username_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="username_obj"><i class="fas fa-user"></i>&nbsp;Username</label>#}
+{#                                                    </div>#}
+
+{#                                                    <li class="list-group-item text-left">#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="CveCheck" name="CveCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="CveCheck"><i class="fas fa-bug"></i> Cve</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="CryptocurrencyCheck" name="CryptocurrencyCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="CryptocurrencyCheck"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="DecodedCheck" name="DecodedCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="DecodedCheck"><i class="fas fa-lock-open"></i> Decoded</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="ScreenshotCheck"><i class="fas fa-image"></i> Screenshot</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="PgpCheck"><i class="fas fa-key"></i> PGP</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="UsernameCheck"><i class="fas fa-user"></i> Username</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="DomainCheck"><i class="fas fa-spider"></i> Domain</label>#}
+{#                                                        </div>#}
+{#                                                        <div class="form-check">#}
+{#                                                            <input class="form-check-input" type="checkbox" value="True" id="ItemCheck" name="ItemCheck" checked="">#}
+{#                                                            <label class="form-check-label" for="ItemCheck"><i class="fas fa-user"></i> Item</label>#}
+{#                                                        </div>#}
+{##}
+{#                                                </li>#}
+
+
+												</div>
+											</div>
+
+											<div class="card my-4">
+                                                <div class="card-header bg-secondary text-white">
+													<b>Tags</b>
+												</div>
+                                                <div class="card-body">
+													<div class="input-group mb-2 mr-sm-2">
+														<div class="input-group-prepend">
+													  	    <div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
+													    </div>
+														<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
+													</div>
+													{% include 'tags/block_tags_selector.html' %}
+												</div>
+											</div>
+
+										</div>
+										<div class="col-12 col-xl-3">
+											<div class="custom-control custom-switch mt-1">
+												<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
+												<label class="custom-control-label" for="id_level">
+													<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
+												</label>
+											</div>
+										</div>
+									</div>
+
+									<hr>
+									<h4>Tracker Type:</h4>
+
+									<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
+										<option disabled selected value> -- Select a tracker type -- </option>
+										<option value="word">Word</option>
+										<option value="set">Set</option>
+										<option value="regex">Regex</option>
+										<option value="yara">YARA rule</option>
+										<option value="typosquatting">Typo-squatting</option>
+									</select>
+
+									<p id="tracker_desc">Terms to track (space separated)</p>
+
+									<div class="row" id="simple_input">
+										<div class="col-12 col-lg-10">
+											<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
+										</div>
+										<div class="col-12 col-lg-2">
+											<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
+										</div>
+									</div>
+
+
+									<div class="mb-2" id="yara_rule">
+										<div class="" id="yara_default_rules">
+
+											<h6>Default YARA rules:</h6>
+											<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
+											  <option selected>Select a default rule</option>
+												{% for yara_types in all_yara_files %}
+													{% for yara_file_name in all_yara_files[yara_types] %}
+													<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
+													{% endfor %}
+												{% endfor %}
+											</select>
+
+											<pre class="border bg-light" id="default_yara_rule_content"></pre>
+
+										</div>
+
+										<hr>
+
+										<h6>Custom YARA rules:</h6>
+										<div class="row" id="textarea">
+											<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
+										</div>
+									</div>
+
+									<br>
+									<button class="btn btn-success mt-2">
+										<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
+									</button>
+
+								</form>
+
+
+
+						  </div>
+						</div>
+
+
+			</div>
+
+		</div>
+	</div>
+
+</body>
+
+<script>
+var ltags;
+var chart = {};
+$(document).ready(function(){
+	$("#page-Tracker").addClass("active");
+	$("#nav_manual_crawler").addClass("active");
+	$("#tracker_desc").hide();
+	$("#tracker").hide();
+	$("#nb_word").hide();
+	$("#yara_rule").hide();
+
+
+	sources_item = $('#sources_item').tagSuggest({
+		data: {{all_sources|safe}},
+		{%if dict_tracker%}{%if dict_tracker['item_sources']%}value: {{dict_tracker['item_sources']|safe}},{%endif%}{%endif%}
+		sortOrder: 'name',
+		maxDropHeight: 200,
+		name: 'sources_item',
+		emptyText: 'Item Sources to track (ALL IF EMPTY)',
+		});
+
+	$('#tracker_type').on('change', function() {
+		var tracker_type = this.value;
+		if (tracker_type=="word") {
+			$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
+			$("#tracker_desc").show();
+			$("#tracker").show();
+			$("#nb_word").hide();
+			$("#yara_rule").hide();
+		} else if (tracker_type=="set") {
+			$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
+			$("#tracker_desc").show();
+			$("#tracker").show();
+			$("#nb_word").show();
+			$("#yara_rule").hide();
+		} else if (tracker_type=="regex") {
+			$("#tracker_desc").text("Enter a valid Python regex");
+			$("#tracker_desc").show();
+			$("#tracker").show();
+			$("#nb_word").hide();
+			$("#yara_rule").hide();
+		} else if (tracker_type=="yara") {
+			$("#tracker_desc").text("Select a default yara rule or create your own rule:");
+			$("#tracker_desc").show();
+			$("#tracker").hide();
+			$("#nb_word").hide();
+			$("#yara_rule").show();
+		} else if (tracker_type=="typosquatting") {
+			$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
+			$("#tracker_desc").show();
+			$("#tracker").show();
+			$("#nb_word").hide();
+			$("#yara_rule").hide();
+		}
+	});
+
+	{%if dict_tracker%}
+		$('#tracker_type').val('{{dict_tracker['type']}}').change();
+
+		{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
+			$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
+		{%endif%}
+	{%endif%}
+
+    $('#item_obj').on("change", function () {
+		item_source_input_controller();
+	});
+    $('#pgp_obj').on("change", function () {
+		pgp_source_input_controller();
+	});
+
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+function item_source_input_controller() {
+	if($('#item_obj').is(':checked')){
+		$("#sources_item_div").show();
+	}else{
+		$("#sources_item_div").hide();
+	}
+}
+
+function pgp_source_input_controller() {
+	if($('#pgp_obj').is(':checked')){
+		$("#sources_pgp_div").show();
+	}else{
+		$("#sources_pgp_div").hide();
+	}
+}
+
+
+function get_default_rule_content(selector){
+	var yara_name = selector.value
+	if (yara_name === "Select a default rule") {
+		jQuery("#default_yara_rule_content").text("")
+	} else {
+		$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
+			function(data) {
+				jQuery("#default_yara_rule_content").text(data['content'])
+		});
+	}
+}
+
+</script>

var/www/templates/hunter/trackersManagement.html

@@ -71,11 +71,11 @@
                                 <td>
                       <span>
                         <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
-                        {% if dict_uuid['tracker'] %}
+                        {% if dict_uuid['tracked'] %}
-                            {% if dict_uuid['tracker']|length > 256 %}
+                            {% if dict_uuid['tracked']|length > 256 %}
-                                {{ dict_uuid['tracker'][0:256] }}...
+                                {{ dict_uuid['tracked'][0:256] }}...
                             {% else %}
-                                {{ dict_uuid['tracker'] }}
+                                {{ dict_uuid['tracked'] }}
                             {% endif %}
                         {% endif %}
                         </a>
@@ -136,11 +136,11 @@
                                 <td>
                       <span>
                         <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
-                        {% if dict_uuid['tracker'] %}
+                        {% if dict_uuid['tracked'] %}
-                            {% if dict_uuid['tracker']|length > 256 %}
+                            {% if dict_uuid['tracked']|length > 256 %}
-                                {{ dict_uuid['tracker'][0:256] }}...
+                                {{ dict_uuid['tracked'][0:256] }}...
                             {% else %}
-                                {{ dict_uuid['tracker'] }}
+                                {{ dict_uuid['tracked'] }}
                             {% endif %}
                         {% endif %}
                         </a>
@@ -178,7 +178,7 @@
             </div>
 
 
-            <a class="btn btn-info my-4" href="{{ url_for('hunter.add_tracked_menu') }}">
+            <a class="btn btn-info my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
                 <i class="fas fa-plus-circle ml-auto"></i>
                 Create New Tracker
             </a>

var/www/templates/hunter/trackers_dashboard.html

@@ -0,0 +1,174 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+    <title>Trackers - AIL</title>
+    <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+    <!-- Core CSS -->
+    <link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+    <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js') }}"></script>
+
+</head>
+<body>
+
+{% include 'nav_bar.html' %}
+
+<div class="container-fluid">
+    <div class="row">
+
+        {% include 'hunter/menu_sidebar.html' %}
+
+        <div class="col-12 col-lg-10" id="core_content">
+
+                <div class="d-flex justify-content-around my-2">
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu') }}">
+                        <div class="card-body">
+                            <h5 class="card-title"><i class="fas fa-ruler-combined"></i>
+                                All Trackers&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['all'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_word') }}">
+                        <div class="card-body">
+                            <h5 class="card-title"><i class="fas fa-font"></i>
+                                Words&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['word'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_set') }}">
+                        <div class="card-body">
+                            <h5 class="card-title"><i class="fas fa-layer-group"></i>
+                                Set&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['set'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                </div>
+
+                <div class="d-flex justify-content-around my-2">
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_regex') }}">
+                        <div class="card-body">
+                            <h5 class="card-title">
+                                <i class="fas fa-drafting-compass"></i>
+                                Regex&nbsp;&nbsp;&nbsp;&nbsp;
+                                <span class="badge badge-light">{{ stats['regex'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_typosquatting') }}">
+                        <div class="card-body">
+                            <h5 class="card-title">
+                                <i class="fa fa-clone"></i>
+                                Typo Squatting&nbsp;&nbsp;&nbsp;&nbsp;
+                                <span class="badge badge-light">{{ stats['typosquatting'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                    <a class="btn btn-info  border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_yara') }}">
+                        <div class="card-body">
+                            <h5 class="card-title">
+                                <span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
+                                Yara&nbsp;&nbsp;&nbsp;&nbsp;
+                                <span class="badge badge-light">{{ stats['yara'] }}</span>
+                            </h5>
+                        </div>
+                    </a>
+                </div>
+
+        {# TODO Buttons#}
+
+            <div class="card my-3">
+                <div class="card-header bg-dark text-white">
+                    <h5 class="card-title"><b>Trackers Last Matches</b></h5>
+                </div>
+                <div class="card-body">
+                    <table id="table_global_trackers" class="table table-striped table-borderless table-condensed">
+                        <tbody style="font-size: 15px;">
+                        {% for meta in trackers %}
+                            <tr>
+                                <td>{{ meta['type'] }}</td>
+                                <td>
+                                    <span>
+                                        <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ meta['uuid'] }}">
+                                            {% if meta['tracked'] %}
+                                                {% if meta['tracked']|length > 256 %}
+                                                    {{ meta['tracked'][0:256] }}...
+                                                {% else %}
+                                                    {{ meta['tracked'] }}
+                                                {% endif %}
+                                            {% endif %}
+                                        </a>
+                                    </span>
+                                </td>
+                                <td>
+                                    {% for tag in meta['tags'] %}
+                                        <a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        </a>
+                                    {% endfor %}
+                                </td>
+                                <td>
+                                    {% if meta['first_seen'] %}
+                                        {{ meta['first_seen'][0:4] }}/{{ meta['first_seen'][4:6] }}/{{ meta['first_seen'][6:8] }}
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    {{ meta['timestamp'] }}
+                                </td>
+{#                                <td id="sparklines_{{ dict_uuid['uuid'] }}" style="text-align:center;"></td>#}
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+
+
+            <a class="btn btn-primary my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
+                <i class="fas fa-plus-circle ml-auto"></i>
+                Create New Tracker
+            </a>
+
+        </div>
+    </div>
+</div>
+
+<script>
+    $(document).ready(function () {
+        $("#page-Tracker").addClass("active");
+        $("#nav_tracker_{{tracker_type}}").addClass("active");
+
+{#        {% for dict_uuid in global_trackers %}#}
+{#            sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});#}
+{#        {% endfor %}#}
+
+    });
+
+    function toggle_sidebar() {
+        if ($('#nav_menu').is(':visible')) {
+            $('#nav_menu').hide();
+            $('#side_menu').removeClass('border-right')
+            $('#side_menu').removeClass('col-lg-2')
+            $('#core_content').removeClass('col-lg-10')
+        } else {
+            $('#nav_menu').show();
+            $('#side_menu').addClass('border-right')
+            $('#side_menu').addClass('col-lg-2')
+            $('#core_content').addClass('col-lg-10')
+        }
+    }
+</script>
+
+</body>
+</html>

var/www/templates/nav_bar.html

@@ -19,7 +19,7 @@
         <a class="nav-link" id="navbar-tags" href="{{ url_for('tags_ui.tags_search_items') }}" aria-disabled="true"><i class="fas fa-tag"></i> Tags</a>
       </li>
       <li class="nav-item mr-3">
-        <a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
+        <a class="nav-link" id="page-Tracker" href="{{ url_for('hunters.trackers_dashboard') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
       </li>
       <li class="nav-item mr-3">
         <a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>