chg: [objects + retro hunt] refactor retro hunt + objects retro hunts + get objects generator by filters (date_from, sources, mimetypes, ...)

bin/lib/ail_core.py

@@ -43,6 +43,9 @@ def get_object_all_subtypes(obj_type):
 def get_objects_tracked():
     return ['decoded', 'item', 'pgp']
 
+def get_objects_retro_hunted():
+    return ['decoded', 'item']
+
 def get_all_objects_with_subtypes_tuple():
     str_objs = []
     for obj_type in get_all_objects():
@@ -56,6 +59,21 @@ def get_all_objects_with_subtypes_tuple():
 
 ##-- AIL OBJECTS --##
 
+####    Redis     ####
+
+def _parse_zscan(response):
+    cursor, r = response
+    it = iter(r)
+    return str(cursor), list(it)
+
+def zscan_iter(r_redis, name):  # count ???
+    cursor = 0
+    while cursor != "0":
+        cursor, data = _parse_zscan(r_redis.zscan(name, cursor=cursor))
+        yield from data
+
+## --    Redis     -- ##
+
 def paginate_iterator(iter_elems, nb_obj=50, page=1):
     dict_page = {'nb_all_elem': len(iter_elems)}
     nb_pages = dict_page['nb_all_elem'] / nb_obj

bin/lib/objects/Decodeds.py

@@ -372,6 +372,67 @@ def search_decodeds_by_name(name_to_search, r_pos=False):
                 decodeds[decoded_name]['hl-end'] = res.end()
     return decodeds
 
+
+############################################################################
+
+def get_decodeds_dir():
+    decodeds_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR)
+    if not decodeds_dir.endswith("/"):
+        decodeds_dir = f"{decodeds_dir}/"
+    return decodeds_dir
+
+# Generator
+
+def get_nb_decodeds_objects(filters={}):
+    nb = 0
+    if 'mimetypes' in filters:
+        mimetypes = filters['mimetypes']
+    else:
+        mimetypes = get_all_mimetypes()
+    d_dir = get_decodeds_dir()
+    for mimetype in mimetypes:
+        for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
+            nb += len(files)
+    return nb
+
+def get_all_decodeds_objects(filters={}):
+    if 'mimetypes' in filters:
+        # TODO sanityze mimetype
+        mimetypes = filters['mimetypes']
+    else:
+        mimetypes = get_all_mimetypes()
+    mimetypes = sorted(mimetypes)
+
+    if filters.get('start'):
+        _, start_id = filters['start'].split(':', 1)
+        decoded = Decoded(start_id)
+        # remove sources
+        start_mimetype = decoded.get_mimetype()
+        i = 0
+        while start_mimetype and len(mimetypes) > i:
+            if mimetypes[i] == start_mimetype:
+                mimetypes = mimetypes[i:]
+                start_mimetype = None
+            i += 1
+    else:
+        start_id = None
+
+    d_dir = get_decodeds_dir()
+    for mimetype in mimetypes:
+        for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
+            if start_id:
+                i = 0
+                while start_id and len(files) > i:
+                    if files[i] == start_id:
+                        files = files[i:]
+                        start_id = None
+                    i += 1
+                if i >= len(files):
+                    files = []
+            for file in files:
+                yield Decoded(file).id
+
+
 ############################################################################
 
 def sanityze_decoder_names(decoder_name):
@@ -538,6 +599,12 @@ def get_all_decodeds_files():
     return decodeds
 
 
-# if __name__ == '__main__':
+if __name__ == '__main__':
 #     name_to_search = '4d36'
 #     print(search_decodeds_by_name(name_to_search))
+#     filters = {'mimetypes': ['text/html']}
+    filters = {'start': ':1a005f82a4ae0940205c8fd81fd14838845696be'}
+    # filters = {}
+    gen = get_all_decodeds_objects(filters=filters)
+    for f in gen:
+        print(f)

bin/lib/objects/Items.py

@@ -22,7 +22,8 @@ from lib.ail_core import get_ail_uuid
 from lib.objects.abstract_object import AbstractObject
 from lib.ConfigLoader import ConfigLoader
 from lib import item_basic
-from lib.data_retention_engine import update_obj_date
+from lib.data_retention_engine import update_obj_date, get_obj_date_first
+from packages import Date
 
 
 from flask import url_for
@@ -406,6 +407,115 @@ def _manual_set_items_date_first_last():
     if last != 0:
         update_obj_date(last, 'item')
 
+################################################################################
+################################################################################
+################################################################################
+
+def get_nb_items_objects(filters={}):
+    nb = 0
+    date_from = filters.get('date_from')
+    date_to = filters.get('date_to')
+    if 'sources' in filters:
+        sources = filters['sources']
+    else:
+        sources = get_all_sources()
+    sources = sorted(sources)
+
+    # date
+    if date_from and date_to:
+        daterange = Date.get_daterange(date_from, date_to)
+    elif date_from:
+        daterange = Date.get_daterange(date_from, Date.get_today_date_str())
+    elif date_to:
+        date_from = get_obj_date_first('item')
+        daterange = Date.get_daterange(date_from, date_to)
+    else:
+        date_from = get_obj_date_first('item')
+        daterange = Date.get_daterange(date_from, Date.get_today_date_str())
+
+    for source in sources:
+        for date in daterange:
+            date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
+            full_dir = os.path.join(ITEMS_FOLDER, source, date)
+            if not os.path.isdir(full_dir):
+                continue
+            nb += len(os.listdir(full_dir))
+    return nb
+
+def get_all_items_objects(filters={}):
+    date_from = filters.get('date_from')
+    date_to = filters.get('date_to')
+    if 'sources' in filters:
+        sources = filters['sources']
+    else:
+        sources = get_all_sources()
+    sources = sorted(sources)
+    if filters.get('start'):
+        _, start_id = filters['start'].split(':', 1)
+        item = Item(start_id)
+        # remove sources
+        start_source = item.get_source()
+        i = 0
+        while start_source and len(sources) > i:
+            if sources[i] == start_source:
+                sources = sources[i:]
+                start_source = None
+            i += 1
+        start_date = item.get_date()
+    else:
+        start_id = None
+        start_date = None
+
+    # date
+    if date_from and date_to:
+        daterange = Date.get_daterange(date_from, date_to)
+    elif date_from:
+        daterange = Date.get_daterange(date_from, Date.get_today_date_str())
+    elif date_to:
+        date_from = get_obj_date_first('item')
+        daterange = Date.get_daterange(date_from, date_to)
+    else:
+        date_from = get_obj_date_first('item')
+        daterange = Date.get_daterange(date_from, Date.get_today_date_str())
+    if start_date:
+        if int(start_date) > int(date_from):
+            i = 0
+            while start_date and len(daterange) > i:
+                if daterange[i] == start_date:
+                    daterange = daterange[i:]
+                    start_date = None
+                i += 1
+
+    for source in sources:
+        for date in daterange:
+            date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
+            full_dir = os.path.join(ITEMS_FOLDER, source, date)
+            s_dir = os.path.join(source, date)
+            if not os.path.isdir(full_dir):
+                continue
+
+            # TODO replace by os.scandir() ????
+            all_items = sorted([os.path.join(s_dir, f)
+                                for f in os.listdir(full_dir)
+                                if os.path.isfile(os.path.join(full_dir, f))])
+            # start obj id
+            if start_id:
+                i = 0
+                while start_id and len(all_items) > i:
+                    if all_items[i] == start_id:
+                        if i == len(all_items):
+                            all_items = []
+                        else:
+                            all_items = all_items[i+1:]
+                        start_id = None
+                    i += 1
+            for obj_id in all_items:
+                yield Item(obj_id)
+
+################################################################################
+################################################################################
+################################################################################
+
 #### API ####
 
 def api_get_item(data):
@@ -810,9 +920,13 @@ def create_item(obj_id, obj_metadata, io_content):
 #         delete_item(child_id)
 
 
-# if __name__ == '__main__':
+if __name__ == '__main__':
 #     content = 'test file content'
 #     duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]}
 #
 #     item = Item('tests/2020/01/02/test_save.gz')
 #     item.create(content, _save=False)
+    filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'}
+    gen = get_all_items_objects(filters=filters)
+    for obj_id in gen:
+        print(obj_id.id)

bin/lib/objects/Pgps.py

@@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id, get_all_id_iterator
 
 config_loader = ConfigLoader()
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@@ -128,8 +128,22 @@ def search_pgps_by_name(name_to_search, subtype, r_pos=False):
                 pgps[pgp_name]['hl-end'] = res.end()
     return pgps
 
+def get_all_pgps_objects(filters={}):
+    if 'subtypes' in filters:
+        subtypes = filters['subtypes']
+    else:
+        subtypes = get_all_subtypes()
+    for subtype in subtypes:
+        for z_tuple in get_all_id_iterator('pgp', subtype):
+            obj_id, _ = z_tuple
+            yield Pgp(obj_id, subtype)
 
-# if __name__ == '__main__':
+
+if __name__ == '__main__':
 #     name_to_search = 'ex'
 #     subtype = 'name'
 #     print(search_pgps_by_name(name_to_search, subtype))
+    gen = get_all_pgps_objects(filters={'subtypes': ['key']})
+    for f in gen:
+        print(f)
+

bin/lib/objects/abstract_subtype_object.py

@@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.objects.abstract_object import AbstractObject
-from lib.ail_core import get_object_all_subtypes
+from lib.ail_core import get_object_all_subtypes, zscan_iter
 from lib.ConfigLoader import ConfigLoader
 from lib.item_basic import is_crawled, get_item_domain
 from lib.data_retention_engine import update_obj_date
@@ -176,6 +176,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
 def get_all_id(obj_type, subtype):
     return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
 
+def get_all_id_iterator(obj_type, subtype):
+    return zscan_iter(r_object, f'{obj_type}_all:{subtype}')
+
 def get_subtypes_objs_by_date(obj_type, subtype, date):
     return r_object.hkeys(f'{obj_type}:{subtype}:{date}')
 

bin/lib/objects/ail_objects.py

@@ -9,16 +9,16 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple
+from lib.ail_core import get_all_objects, get_object_all_subtypes
 from lib import correlations_engine
 from lib import btc_ail
 from lib import Tag
 
 from lib.objects import CryptoCurrencies
 from lib.objects.Cves import Cve
-from lib.objects.Decodeds import Decoded
+from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
 from lib.objects.Domains import Domain
-from lib.objects.Items import Item
+from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
 from lib.objects import Pgps
 from lib.objects.Screenshots import Screenshot
 from lib.objects import Usernames
@@ -114,6 +114,9 @@ def get_obj_tags(obj_type, subtype, id):
     obj = get_object(obj_type, subtype, id)
     return obj.get_tags()
 
+def is_obj_tags_safe(obj_type, subtype, id):
+    obj = get_object(obj_type, subtype, id)
+    return obj.is_tags_safe()
 
 def add_obj_tag(obj_type, subtype, id, tag):
     obj = get_object(obj_type, subtype, id)
@@ -143,6 +146,10 @@ def get_objects_meta(objs, options=set(), flask_context=False):
             obj_type = obj['type']
             subtype = obj['subtype']
             obj_id = obj['id']
+        elif isinstance(obj, tuple):
+            obj_type = obj[0]
+            subtype = obj[1]
+            obj_id = obj[2]
         else:
             obj_type, subtype, obj_id = obj.split(':', 2)
         metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
@@ -184,6 +191,27 @@ def is_filtered(obj, filters):
             return True
     return False
 
+def obj_iterator(obj_type, filters):
+    if obj_type == 'decoded':
+        return get_all_decodeds_objects(filters=filters)
+    elif obj_type == 'item':
+        return get_all_items_objects(filters=filters)
+    elif obj_type == 'pgp':
+        return Pgps.get_all_pgps_objects(filters=filters)
+
+def card_objs_iterators(filters):
+    nb = 0
+    for obj_type in filters:
+        nb += int(card_obj_iterator(obj_type, filters.get(obj_type, {})))
+    return nb
+
+def card_obj_iterator(obj_type, filters):
+    if obj_type == 'decoded':
+        return get_nb_decodeds_objects(filters=filters)
+    elif obj_type == 'item':
+        return get_nb_items_objects(filters=filters)
+    elif obj_type == 'pgp':
+        return Pgps.nb_all_pgps_objects(filters=filters)
 
 def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
     """

bin/packages/Date.py

@@ -79,7 +79,10 @@ class Date(object):
         comp_day = str(computed_date.day).zfill(2)
         return comp_year + comp_month + comp_day
 
-def get_today_date_str():
+def get_today_date_str(separator=False):
+    if separator:
+        datetime.date.today().strftime("%Y/%m/%d")
+    else:
         return datetime.date.today().strftime("%Y%m%d")
 
 def date_add_day(date, num_day=1):

bin/trackers/Retro_Hunt.py

@@ -19,149 +19,122 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
+from lib.ail_core import get_objects_retro_hunted
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.Items import Item
+from lib.objects import ail_objects
-from packages import Date
 from lib import Tracker
 
-import NotificationHelper # # TODO: refractor
+class Retro_Hunt_Module(AbstractModule):
-
-class Retro_Hunt(AbstractModule):
-
-    # mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
 
     """
     Retro_Hunt module for AIL framework
     """
     def __init__(self):
-        super(Retro_Hunt, self).__init__()
+        super(Retro_Hunt_Module, self).__init__()
         config_loader = ConfigLoader()
         self.pending_seconds = 5
 
-        self.full_item_url = config_loader.get_config_str("Notifications", "ail_domain") + "/object/item?id="
-
         # reset on each loop
-        self.task_uuid = None
+        self.retro_hunt = None
-        self.date_from = 0
+        self.nb_objs = 0
-        self.date_to = 0
+        self.nb_done = 0
-        self.nb_src_done = 0
         self.progress = 0
-        self.item = None
+        self.obj = None
         self.tags = []
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
-    # # TODO: send mails
+    # # TODO:   # start_time
-    # # TODO:   # start_time # end_time
+    #           # end_time
-
     def compute(self, task_uuid):
         self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
         print(f'starting Retro hunt task {task_uuid}')
-        self.task_uuid = task_uuid
         self.progress = 0
         # First launch
         # restart
-        retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
+        self.retro_hunt = Tracker.RetroHunt(task_uuid)
 
-        rule = retro_hunt.get_rule(r_compile=True)
+        rule = self.retro_hunt.get_rule(r_compile=True)
+        timeout = self.retro_hunt.get_timeout()
+        self.tags = self.retro_hunt.get_tags()
 
-        timeout = retro_hunt.get_timeout()
         self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
-        sources = retro_hunt.get_sources(r_sort=True)
 
-        self.date_from = retro_hunt.get_date_from()
+        # Filters
-        self.date_to = retro_hunt.get_date_to()
+        filters = self.retro_hunt.get_filters()
-        self.tags = retro_hunt.get_tags()
+        if not filters:
-        curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
+            filters = {}
-        self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
+            for obj_type in get_objects_retro_hunted():
-        self.update_progress(sources, curr_date)
+                filters[obj_type] = {}
-        # iterate on date
-        filter_last = True
-        while int(curr_date) <= int(self.date_to):
-            print(curr_date)
-            dirs_date = Tracker.get_retro_hunt_dir_day_to_analyze(task_uuid, curr_date, filter_last=filter_last, sources=sources)
-            filter_last = False
-            nb_id = 0
-            self.nb_src_done = 0
-            self.update_progress(sources, curr_date)
-            # # TODO: Filter previous item
-            for dir in dirs_date:
-                print(dir)
-                self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
-                l_obj = Tracker.get_items_to_analyze(dir)
-                for id in l_obj:
-                    # print(f'{dir} / {id}')
-                    self.item = Item(id)
-                    # save current item in cache
-                    Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
 
-                    self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
+        self.nb_objs = ail_objects.card_objs_iterators(filters)
 
-                    yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match,
+        # Resume
+        last_obj = self.retro_hunt.get_last_analyzed()
+        if last_obj:
+            last_obj_type, last_obj_subtype, last_obj_id = last_obj.split(':', 2)
+        else:
+            last_obj_type = None
+            last_obj_subtype = None
+            last_obj_id = None
+
+        self.nb_done = 0
+        self.update_progress()
+
+        for obj_type in filters:
+            if last_obj_type:
+                filters['start'] = f'{last_obj_subtype}:{last_obj_id}'
+                last_obj_type = None
+            for obj in ail_objects.obj_iterator(obj_type, filters):
+                self.obj = obj
+                content = obj.get_content(r_str=True)
+                rule.match(data=content, callback=self.yara_rules_match,
                            which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
 
-                    # save last item
+                self.nb_done += 1
-                    if nb_id % 10 == 0: # # TODO: Add nb before save in DB
+                if self.nb_done % 10 == 0:
-                        Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
+                    self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
-                    nb_id += 1
+                self.retro_hunt.set_last_analyzed_cache(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
-                    self.update_progress(sources, curr_date)
+
+                # update progress
+                self.update_progress()
 
                 # PAUSE
-                    self.update_progress(sources, curr_date)
+                if self.retro_hunt.to_pause():
-                    if retro_hunt.to_pause():
+                    self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
-                        Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
+                    self.retro_hunt.pause()
-                        # self.update_progress(sources, curr_date, save_db=True)
-                        retro_hunt.pause()
                     return None
 
-                self.nb_src_done += 1
+        # Completed
-                self.update_progress(sources, curr_date)
+        self.retro_hunt.complete()
-            curr_date = Date.date_add_day(curr_date)
-            print('-----')
-
-        self.update_progress(sources, curr_date)
-
-        retro_hunt.complete()
-
         print(f'Retro Hunt {task_uuid} completed')
         self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
 
-        # # TODO: stop
+    def update_progress(self):
-
+        new_progress = self.nb_done * 100 / self.nb_objs
-    def update_progress(self, sources, curr_date, save_db=False):
+        if int(self.progress) != int(new_progress):
-        retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
+            print(new_progress)
-        progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
+            self.retro_hunt.set_progress(new_progress)
-                                               sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
+            self.progress = new_progress
-        if self.progress != progress:
-            retro_hunt.set_progress(progress)
-            self.progress = progress
-        # if save_db:
-        #     Tracker.set_retro_hunt_task_progress(task_uuid, progress)
 
     def yara_rules_match(self, data):
-        id = self.item.get_id()
+        obj_id = self.obj.get_id()
         # print(data)
         task_uuid = data['namespace']
 
-        self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found:    {id}')
+        self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found:    {obj_id}')
-        print(f'Retro hunt {task_uuid} match found:    {id}')
+        print(f'Retro hunt {task_uuid} match found:   {self.obj.get_type()} {obj_id}')
 
-        Tracker.save_retro_hunt_match(task_uuid, id)
+        self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(), obj_id)
 
+        # TODO FILTER Tags
         # Tags
         for tag in self.tags:
             msg = f'{tag};{id}'
             self.add_message_to_queue(msg, 'Tags')
 
         # # Mails
-        # mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
+        # EXPORTER MAILS
-        # if mail_to_notify:
-        #     mail_subject = Tracker.get_email_subject(tracker_uuid)
-        #     mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
-        # for mail in mail_to_notify:
-        #     self.redis_logger.debug(f'Send Mail {mail_subject}')
-        #     print(f'Send Mail {mail_subject}')
-        #     NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
         return yara.CALLBACK_CONTINUE
 
     def run(self):
@@ -188,6 +161,5 @@ class Retro_Hunt(AbstractModule):
 
 
 if __name__ == '__main__':
-
+    module = Retro_Hunt_Module()
-    module = Retro_Hunt()
     module.run()

configs/modules.cfg

@@ -40,7 +40,7 @@ publish = D4_client
 [D4Client]
 subscribe = D4_client
 
-[Retro_Hunt]
+[Retro_Hunt_Module]
 publish = Tags
 
 [Tracker_Typo_Squatting]

var/www/blueprints/hunters.py

@@ -9,7 +9,7 @@ import os
 import sys
 import json
 
-from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape
 from flask_login import login_required, current_user, login_user, logout_user
 
 sys.path.append('modules')
@@ -23,6 +23,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib import ail_core
+from lib.objects import ail_objects
 from lib import item_basic
 from lib import Tracker
 from lib import Tag
@@ -149,6 +150,8 @@ def add_tracked_menu():
                 galaxies_tags = json.loads(galaxies_tags)
             except:
                 galaxies_tags = []
+        else:
+            galaxies_tags = []
         # custom tags
         if tags:
             tags = tags.split()
@@ -242,7 +245,7 @@ def tracker_delete():
 @login_required
 @login_read_only
 def retro_hunt_all_tasks():
-    retro_hunts = Tracker.get_retro_hunt_tasks_metas()
+    retro_hunts = Tracker.get_retro_hunt_metas()
     return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
 
 @hunters.route('/retro_hunt/task/show', methods=['GET'])
@@ -250,40 +253,35 @@ def retro_hunt_all_tasks():
 @login_read_only
 def retro_hunt_show_task():
     task_uuid = request.args.get('uuid', None)
+    objs = request.args.get('objs', False)
 
-    date_from = request.args.get('date_from')
+    date_from_item = request.args.get('date_from')
-    date_to = request.args.get('date_to')
+    date_to_item = request.args.get('date_to')
-    if date_from:
+    if date_from_item:
-        date_from = date_from.replace('-', '')
+        date_from_item = date_from_item.replace('-', '')
-    if date_to:
+    if date_to_item:
-        date_to = date_to.replace('-', '')
+        date_to_item = date_to_item.replace('-', '')
 
     res = Tracker.api_check_retro_hunt_task_uuid(task_uuid)
     if res:
         return create_json_response(res[0], res[1])
 
     retro_hunt = Tracker.RetroHunt(task_uuid)
-    dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
+    dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'filters', 'nb_objs', 'tags'})
     rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
+    dict_task['filters'] = json.dumps(dict_task['filters'], indent=4)
 
-    if date_from:
+    if objs:
-        res = Tracker.api_get_retro_hunt_items({'uuid': task_uuid, 'date_from': date_from, 'date_to': date_to})
+        dict_task['objs'] = ail_objects.get_objects_meta(retro_hunt.get_objs(), flask_context=True)
-        if res[1] != 200:
-            return create_json_response(res[0], res[1])
-        dict_task['items'] = res[0]['items']
-        dict_task['date_from_input'] = res[0]['date_from']
-        dict_task['date_to_input'] = res[0]['date_to']
     else:
-        dict_task['items'] = []
+        dict_task['objs'] = []
-        dict_task['date_from_input'] = dict_task['date_from']
-        dict_task['date_to_input'] = dict_task['date_to']
 
     return render_template("show_retro_hunt.html", dict_task=dict_task,
                            rule_content=rule_content,
                            bootstrap_label=bootstrap_label)
 
 
-@hunters.route('/retro_hunt/task/add', methods=['GET', 'POST'])
+@hunters.route('/retro_hunt/add', methods=['GET', 'POST'])
 @login_required
 @login_analyst
 def retro_hunt_add_task():
@@ -291,23 +289,69 @@ def retro_hunt_add_task():
         name = request.form.get("name", '')
         description = request.form.get("description", '')
         timeout = request.form.get("timeout", 30)
+        # TAGS
         tags = request.form.get("tags", [])
+        taxonomies_tags = request.form.get('taxonomies_tags')
+        if taxonomies_tags:
+            try:
+                taxonomies_tags = json.loads(taxonomies_tags)
+            except:
+                taxonomies_tags = []
+        else:
+            taxonomies_tags = []
+        galaxies_tags = request.form.get('galaxies_tags')
+        if galaxies_tags:
+            try:
+                galaxies_tags = json.loads(galaxies_tags)
+            except:
+                galaxies_tags = []
+        else:
+            galaxies_tags = []
+        # custom tags
         if tags:
             tags = tags.split()
+            escaped_tags = []
+            for tag in tags:
+                escaped_tags.append(escape(tag))
+            tags = escaped_tags
+        else:
+            tags = []
+        tags = tags + taxonomies_tags + galaxies_tags
         # mails = request.form.get("mails", [])
         # if mails:
         #     mails = mails.split()
 
-        sources = request.form.get("sources", [])
+        # FILTERS
+        filters = {}
+        for obj_type in Tracker.get_objects_tracked():
+            new_filter = request.form.get(f'{obj_type}_obj')
+            if new_filter == 'on':
+                filters[obj_type] = {}
+                # Date From
+                date_from = request.form.get(f'date_from_{obj_type}', '').replace('-', '')
+                if date_from:
+                    filters[obj_type]['date_from'] = date_from
+                # Date to
+                date_to = request.form.get(f'date_to_{obj_type}', '').replace('-', '')
+                if date_to:
+                    filters[obj_type]['date_to'] = date_to
+                # Mimetypes
+                mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
+                if mimetypes:
+                    mimetypes = json.loads(mimetypes)
+                    filters[obj_type]['mimetypes'] = mimetypes
+                # Sources
+                sources = request.form.get(f'sources_{obj_type}', [])
                 if sources:
                     sources = json.loads(sources)
-
+                    filters[obj_type]['sources'] = sources
-        date_from = request.form.get('date_from')
+                # Subtypes
-        date_to = request.form.get('date_to')
+                for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
-        if date_from:
+                    subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
-            date_from = date_from.replace('-', '')
+                    if subtype == 'on':
-        if date_to:
+                        if 'subtypes' not in filters[obj_type]:
-            date_to = date_to.replace('-', '')
+                            filters[obj_type]['subtypes'] = []
+                        filters[obj_type]['subtypes'].append(obj_subtype)
 
         # YARA #
         yara_default_rule = request.form.get("yara_default_rule")
@@ -323,8 +367,8 @@ def retro_hunt_add_task():
 
         input_dict = {"name": name, "description": description, "creator": user_id,
                       "rule": rule, "type": rule_type,
-                        "tags": tags, "sources": sources, "timeout": timeout, #"mails": mails,
+                      "tags": tags, "filters": filters, "timeout": timeout,  # "mails": mails
-                        "date_from": date_from, "date_to": date_to}
+                      }
 
         res = Tracker.api_create_retro_hunt_task(input_dict, user_id)
         if res[1] == 200:
@@ -335,7 +379,8 @@ def retro_hunt_add_task():
     else:
         return render_template("add_retro_hunt_task.html",
                                all_yara_files=Tracker.get_all_default_yara_files(),
-                                all_sources=item_basic.get_all_items_sources(r_list=True))
+                               tags_selector_data=Tag.get_tags_selector_data(),
+                               items_sources=item_basic.get_all_items_sources(r_list=True))
 
 @hunters.route('/retro_hunt/task/pause', methods=['GET'])
 @login_required
@@ -368,27 +413,4 @@ def retro_hunt_delete_task():
     return redirect(url_for('hunters.retro_hunt_all_tasks'))
 
 
-#### JSON ####
-
-@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
-@login_required
-@login_read_only
-def get_json_retro_hunt_nb_items_by_date():
-    date_from = request.args.get('date_from')
-    date_to = request.args.get('date_to')
-
-    if date_from:
-        date_from = date_from.replace('-', '')
-    if date_to:
-        date_to = date_to.replace('-', '')
-
-    task_uuid = request.args.get('uuid')
-
-    if date_from and date_to:
-        res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to)
-    else:
-        res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid])
-    return jsonify(res)
-
-
 ##  - -  ##

var/www/modules/hunter/Flask_hunter.py

@@ -127,6 +127,7 @@ def show_tracker():
     meta['date_to'] = date_to
     print(meta['filters'])
     meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
+    # meta['filters'] = json.dumps(meta['filters'], indent=4)
 
     return render_template("showTracker.html", tracker_metadata=meta,
                            yara_rule_content=yara_rule_content,

var/www/templates/hunter/add_retro_hunt_task.html

@@ -47,12 +47,6 @@
                                         </div>
                                         <input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required>
                                     </div>
-											<div class="input-group mb-2 mr-sm-2">
-										    <div class="input-group-prepend">
-										      <div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
-										    </div>
-												<input id="tags" name="tags" class="form-control" placeholder="Tags to add on match (optional, space separated)" type="text">
-										  </div>
 
 											<!-- <div class="input-group mb-2 mr-sm-2">
 										    <div class="input-group-prepend">
@@ -67,28 +61,112 @@
                                         <textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea>
                                     </div>
 
+                                    <div class="card my-4">
+                                        <div class="card-header bg-info text-white">
+                                            <b>Objects to Track:</b>
+                                        </div>
+                                        <div class="card-body">
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i>&nbsp;CVE</label>#}
+{#                                                    </div>#}
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
+{#                                                    </div>#}
+                                            <div class="custom-control custom-switch mt-1">
+                                                <input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj">
+                                                <label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
+                                            </div>
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
+{#                                                        <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
+{#                                                    </div>#}
+                                            <div class="custom-control custom-switch mt-1">
+                                                <input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
+                                                <label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item</label>
+                                            </div>
+                                            <div class="card border-dark mb-4" id="sources_item_div">
+                                                <div class="card-body">
+                                                    <h5>Filter Item by sources</h5>
                                                     <div class="input-group mb-2 mr-sm-2">
                                                         <div class="input-group-prepend">
                                                             <div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
                                                         </div>
-												<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
+                                                        <input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
                                                     </div>
-
+                                                    <h5>Date range:</h5>
-											<h6>Date range:</h6>
                                                     <div class="row mb-2">
                                                         <div class="col-lg-6">
                                                             <div class="input-group" id="date-range-from">
                                                                 <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-														<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off" required>
+                                                                <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from_item" autocomplete="off">
                                                             </div>
                                                         </div>
                                                         <div class="col-lg-6">
                                                             <div class="input-group" id="date-range-to">
                                                                 <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-														<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off" required>
+                                                                <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to_item" autocomplete="off">
                                                             </div>
                                                         </div>
                                                     </div>
+                                                </div>
+                                            </div>
+
+
+{#                                            <div class="custom-control custom-switch mt-1">#}
+{#                                                <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">#}
+{#                                                <label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP</label>#}
+{#                                            </div>#}
+{#                                            <div class="card border-dark mb-4" id="sources_pgp_div">#}
+{#                                                <div class="card-body">#}
+{#                                                    <h6>Filter PGP by subtype:</h6>#}
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">#}
+{#                                                        <label class="custom-control-label" for="filter_pgp_name">#}
+{#                                                            <svg height="26" width="26">#}
+{#                                                                <g class="nodes">#}
+{#                                                                    <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
+{#                                                                    <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>#}
+{#                                                                </g>#}
+{#                                                            </svg>#}
+{#                                                            name#}
+{#                                                        </label>#}
+{#                                                    </div>#}
+{#                                                    <div class="custom-control custom-switch mt-1">#}
+{#                                                        <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">#}
+{#                                                        <label class="custom-control-label" for="filter_pgp_mail">#}
+{#                                                            <svg height="26" width="26">#}
+{#                                                                <g class="nodes">#}
+{#                                                                    <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
+{#                                                                    <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px">&#xf1fa;</text>#}
+{#                                                                </g>#}
+{#                                                            </svg>#}
+{#                                                            mail#}
+{#                                                        </label>#}
+{#                                                    </div>#}
+{#                                                </div>#}
+{#                                            </div>#}
+
+
+                                        </div>
+                                    </div>
+
+                                    <div class="card my-4">
+                                        <div class="card-header bg-secondary text-white">
+                                            <b>Tags</b>
+                                        </div>
+                                        <div class="card-body">
+                                            <div class="input-group mb-2 mr-sm-2">
+                                                <div class="input-group-prepend">
+                                                    <div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
+                                                </div>
+                                                <input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text">
+                                            </div>
+                                            {% include 'tags/block_tags_selector.html' %}
+                                        </div>
+                                    </div>
+
 
                                 </div>
                                 <div class="col-12 col-xl-3">
@@ -120,7 +198,7 @@
 
                                 <h6>Custom YARA rules:</h6>
                                 <div class="row" id="textarea">
-											<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
+                                    <textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
                                 </div>
                             </div>
 
@@ -138,7 +216,6 @@
 
 
 			</div>
-
 		</div>
 	</div>
 
@@ -177,22 +254,21 @@ $(document).ready(function(){
 			}
 	});
 
-	sources = $('#sources').tagSuggest({
+	sources_item = $('#sources_item').tagSuggest({
-		data: {{all_sources|safe}},
+		data: {{items_sources|safe}},
-		{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
 		sortOrder: 'name',
 		maxDropHeight: 200,
-		name: 'sources',
+		name: 'sources_item',
-		emptyText: 'Sources to track (ALL IF EMPTY)',
+		emptyText: 'Item Sources to track (ALL IF EMPTY)',
 		});
 
-	{%if dict_tracker%}
+    $('#item_obj').on("change", function () {
-		$('#tracker_type').val('{{dict_tracker['type']}}').change();
+		item_source_input_controller();
+	});
+    /*$('#pgp_obj').on("change", function () {
+		pgp_source_input_controller();
+	});*/
 
-		{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
-			$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
-		{%endif%}
-	{%endif%}
 
 });
 
@@ -210,7 +286,21 @@ function toggle_sidebar(){
 	}
 }
 
+function item_source_input_controller() {
+	if($('#item_obj').is(':checked')){
+		$("#sources_item_div").show();
+	}else{
+		$("#sources_item_div").hide();
+	}
+}
 
+function pgp_source_input_controller() {
+	if($('#pgp_obj').is(':checked')){
+		$("#sources_pgp_div").show();
+	}else{
+		$("#sources_pgp_div").hide();
+	}
+}
 
 function get_default_rule_content(selector){
 	var yara_name = selector.value

var/www/templates/hunter/show_retro_hunt.html

@@ -20,7 +20,6 @@
 		<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 		<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
 		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
-		<script src="{{ url_for('static', filename='js/d3/graphlinesgroup.js')}}"></script>
 		<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
 		<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
 
@@ -68,7 +67,7 @@
 										<h3 class="card-title">
 											{{dict_task['name']}}
 											{%if dict_task['state']!='completed'%}
-												<span class="text-info">[{{ dict_task['progress']}}%]<span>
+												<span class="text-info">[{{ dict_task['progress']}}%]</span>
 											{%endif%}
 										</h3>
 
@@ -102,13 +101,6 @@
                                                 {{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
                                             </td>
                                         </tr>
-												<tr>
-													<td class="text-right"><b>Search date</b></td>
-													<td>
-														{{dict_task['date_from'][0:4]}}/{{dict_task['date_from'][4:6]}}/{{dict_task['date_from'][6:8]}} -
-														{{dict_task['date_to'][0:4]}}/{{dict_task['date_to'][4:6]}}/{{dict_task['date_to'][6:8]}}
-													</td>
-												</tr>
                                         <tr>
                                             <td class="text-right"><b>Description</b></td>
                                             <td>{{dict_task['description']}}</td>
@@ -128,19 +120,28 @@
                                             <td>{{dict_task['creator']}}</td>
                                         </tr>
                                         <tr>
-													<td class="text-right"><b>Sources</b></td>
+                                            <td class="text-right"><b>Filters</b></td>
                                             <td>
                                                 <div class="">
-															{%if not dict_task['sources']%}
+                                                    {% if dict_task['filters'] %}
-																<span class="badge badge-secondary">All Souces</span>
+                                                        <pre>{{ dict_task['filters'] }}</pre>
                                                     {% else %}
-																{%for source in dict_task['sources']%}
+                                                        <span class="badge badge-secondary">No Filters</span><br>
-																	<span class="badge badge-secondary">{{source}}</span>
-																{%endfor%}
                                                     {% endif %}
                                                 </div>
                                             </td>
                                         </tr>
+                                        <tr>
+                                            <td class="text-right"><b>Objects Match</b></td>
+                                            <td>
+                                                {%for obj_type in dict_task['nb_objs']%}
+                                                    <h4><span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">
+                                                        {{ obj_type }}
+                                                        <span class="badge badge-light">{{ dict_task['nb_objs'][obj_type] }}</span>
+                                                    </span></h4>
+                                                {%endfor%}
+                                            </td>
+                                        </tr>
                                         </tbody>
                                     </table>
 					            </div>
@@ -152,25 +153,25 @@
 							<div class="card border-dark my-2">
 								<div class="card-body">
 
-									<div class="row mb-2">
+{#									<div class="row mb-2">#}
-										<div class="col-lg-6">
+{#										<div class="col-lg-6">#}
-											<div class="input-group" id="date-range-from">
+{#											<div class="input-group" id="date-range-from">#}
-												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+{#												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
-												<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
+{#												<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"> #}
-													value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}">
+{#													value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}"#}
-											</div>
+{#											</div>#}
-										</div>
+{#										</div>#}
-										<div class="col-lg-6">
+{#										<div class="col-lg-6">#}
-											<div class="input-group" id="date-range-to">
+{#											<div class="input-group" id="date-range-to">#}
-												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+{#												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
-												<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
+{#												<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"> #}
-													value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}">
+{#													value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}"#}
-											</div>
+{#											</div>#}
-										</div>
+{#										</div>#}
-									</div>
+{#									</div>#}
 
 									<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
-										<i class="fas fa-search"></i> Show Items
+										<i class="fas fa-search"></i> Show Objects
 									</button>
 
 								</div>
@@ -181,37 +182,49 @@
 					    </div>
 					</div>
 
-					{%if dict_task['items']%}
+					{% if dict_task['objs'] %}
 						<div class="mt-4">
-							<table class="table table-bordered table-hover" id="items_table">
+							<table class="table table-bordered table-hover" id="objs_table">
 								<thead class="thead-dark">
 									<tr>
-										<th>Date</th>
+										<th>Type</th>
-										<th>Item Id</th>
+                                        <th></th>
+                                        <th>Id</th>
+                                        <th>Tags</th>
+                                        <th></th>
 									</tr>
 								</thead>
 								<tbody>
-
+									{% for object in dict_task['objs'] %}
-									{% for item in dict_task['items'] %}
+                                        <tr class="border-color: blue;">
-										<tr>
                                             <td>
-												{{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}}
+                                                {% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
+                                                    {% include 'objects/obj_svg_block.html' %}
+                                                {% endwith %}
+                                                {{ object['type']}}
                                             </td>
                                             <td>
-												<a class="text-secondary" target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{item['id']}}">
+                                                {% if object['subtype'] %}
-													<div style="line-height:0.9;">{{ item['id'] }}</div>
+                                                    {{ object['subtype']}}
+                                                {% endif %}
+                                            </td>
+                                            <td>
+                                                <a href="{{ object['link'] }}">
+                                                    {{ object['id']}}
                                                 </a>
-												<div class="mb-2">
+                                            </td>
-					                {% for tag in item['tags'] %}
+                                            <td>
-						                <a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
+                                                {% for tag in object['tags'] %}
                                                     <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
-						                </a>
                                                 {% endfor %}
-					              </div>
+                                            </td>
+                                            <td class="text-right">
+            {#									<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
+            {#										<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
+            {#									</a>#}
                                             </td>
                                         </tr>
 									{%  endfor %}
-
 								</tbody>
 							</table>
 						</div>
@@ -219,13 +232,6 @@
 						<hr>
 					{% endif %}
 
-
-					<div id="graphline" class="text-center"></div>
-					<br>
-					<br>
-					<br>
-
-
 				</div>
 			</div>
 		</div>
@@ -234,7 +240,7 @@
 $(document).ready(function(){
 		$('#nav_title_retro_hunt').removeClass("text-muted");
 
-		$('#date-range-from').dateRangePicker({
+		/*$('#date-range-from').dateRangePicker({
 			separator : ' to ',
 			getValue: function(){
 					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
@@ -259,18 +265,15 @@ $(document).ready(function(){
 						$('#date-range-from-input').val(s1);
 						$('#date-range-to-input').val(s2);
 				}
-		});
+		});*/
 
-		$('#items_table').DataTable({
+        {% if dict_task['objs'] %}
+		$('#objs_table').DataTable({
 	        "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
 	        "iDisplayLength": 10,
 	        "order": [[ 0, "asc" ]]
 	    });
-
+        {% endif%}
-		let div_width = $("#graphline").width();
-		$.getJSON( "{{ url_for('hunters.get_json_retro_hunt_nb_items_by_date') }}?uuid={{ dict_task['uuid'] }}&date_from={{ dict_task['date_from_input'] }}&date_to={{ dict_task['date_to_input'] }}",
-			function( data ) {multilines_group("graphline", data, {"width": div_width});}
-		);
 
 });
 
@@ -291,7 +294,8 @@ function toggle_sidebar(){
 function getItems() {
 	var date_from = $('#date-range-from-input').val();
 	var date_to =$('#date-range-to-input').val();
-	window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
+	{#window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);#}
+    window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&objs=True");
 }
 
 </script>