Merge pull request #449 from CIRCL/tags_v2

Tags v2 - Tagging system refractoring

OVERVIEW.md

@@ -198,8 +198,6 @@ Redis and ARDB overview
 ##### Hset:
 | Key | Field | Value |
 | ------ | ------ | ------ |
-| per_paste_**epoch** | **term** | **nb_seen** |
-| | |
 | tag_metadata:**tag** | first_seen | **date** |
 | tag_metadata:**tag** | last_seen | **date** |
 
@@ -207,13 +205,20 @@ Redis and ARDB overview
 | Key | Value |
 | ------ | ------ |
 | list_tags | **tag** |
+| list_tags:**object_type** | **tag** |
+| list_tags:domain | **tag** |
+||
 | active_taxonomies | **taxonomie** |
 | active_galaxies | **galaxie** |
 | active_tag_**taxonomie or galaxy** | **tag** |
 | synonym_tag_misp-galaxy:**galaxy** | **tag synonym** |
 | list_export_tags | **user_tag** |
+||
 | **tag**:**date** | **paste** |
-
+| **object_type**:**tag** | **object_id** |
+||
+| DB7 |
+| tag:**object_id** | **tag** |
 
 ##### old:
 | Key | Value |

bin/Tags.py

@@ -8,29 +8,11 @@ The Tags Module
 This module create tags.
 
 """
-import redis
-
 import time
-import datetime
 
 from pubsublogger import publisher
 from Helper import Process
-from packages import Paste
+from packages import Tag
-from packages import Item
-
-
-def get_item_date(item_filename):
-    l_directory = item_filename.split('/')
-    return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
-
-def set_tag_metadata(tag, date):
-    # First time we see this tag    ## TODO: filter paste from the paste ?
-    if not server.hexists('tag_metadata:{}'.format(tag), 'first_seen'):
-        server.hset('tag_metadata:{}'.format(tag), 'first_seen', date)
-    # Check and Set tag last_seen
-    last_seen = server.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if last_seen is None or date > last_seen:
-        server.hset('tag_metadata:{}'.format(tag), 'last_seen', date)
 
 if __name__ == '__main__':
 
@@ -45,18 +27,6 @@ if __name__ == '__main__':
     # Setup the I/O queues
     p = Process(config_section)
 
-    server = redis.StrictRedis(
-                host=p.config.get("ARDB_Tags", "host"),
-                port=p.config.get("ARDB_Tags", "port"),
-                db=p.config.get("ARDB_Tags", "db"),
-                decode_responses=True)
-
-    server_metadata = redis.StrictRedis(
-                host=p.config.get("ARDB_Metadata", "host"),
-                port=p.config.get("ARDB_Metadata", "port"),
-                db=p.config.get("ARDB_Metadata", "db"),
-                decode_responses=True)
-
     # Sent to the logging a description of the module
     publisher.info("Tags module started")
 
@@ -71,27 +41,8 @@ if __name__ == '__main__':
             continue
 
         else:
-            tag, path = message.split(';')
+            print(message)
-            # add the tag to the tags word_list
+            tag, item_id = message.split(';')
-            res = server.sadd('list_tags', tag)
-            if res == 1:
-                print("new tags added : {}".format(tag))
-            # add the path to the tag set
-            #curr_date = datetime.date.today().strftime("%Y%m%d")
-            item_date = get_item_date(path)
-            res = server.sadd('{}:{}'.format(tag, item_date), path)
-            if res == 1:
-                print("new paste: {}".format(path))
-                print("   tagged: {}".format(tag))
-                set_tag_metadata(tag, item_date)
-            server_metadata.sadd('tag:{}'.format(path), tag)
 
-            # Domain Object
+            Tag.add_tag("item", tag, item_id)
-            if Item.is_crawled(path) and tag!='infoleak:submission="crawler"':
-                domain = Item.get_item_domain(path)
-                server_metadata.sadd('tag:{}'.format(domain), tag)
-                server.sadd('domain:{}:{}'.format(tag, item_date), domain)
-
-            curr_date = datetime.date.today().strftime("%Y%m%d")
-            server.hincrby('daily_tags:{}'.format(item_date), tag, 1)
             p.populate_set_out(message, 'MISP_The_Hive_feeder')

bin/lib/Correlate_object.py

@@ -23,6 +23,15 @@ config_loader = ConfigLoader.ConfigLoader()
 r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 config_loader = None
 
+def is_valid_object_type(object_type):
+    if object_type in ['domain', 'item', 'image']:
+        return True
+    else:
+        return False
+
+def get_all_objects():
+    return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
+
 def get_all_correlation_names():
     '''
     Return a list of all available correlations
@@ -178,11 +187,21 @@ def get_item_url(correlation_name, value, correlation_type=None):
     elif correlation_name == 'domain':
         endpoint = 'crawler_splash.showDomain'
         url = url_for(endpoint, domain=value)
-    elif correlation_name == 'paste':
+    elif correlation_name == 'item':
+        endpoint = 'showsavedpastes.showsavedpaste'
+        url = url_for(endpoint, paste=value)
+    elif correlation_name == 'paste':                   ### # TODO:  remove me
         endpoint = 'showsavedpastes.showsavedpaste'
         url = url_for(endpoint, paste=value)
     return url
 
+def get_obj_tag_table_keys(object_type):
+    '''
+    Warning: use only in flask (dynamic templates)
+    '''
+    if object_type=="domain":
+        return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
+
 
 def create_graph_links(links_set):
     graph_links_list = []
@@ -310,6 +329,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
 
 
 ######## API EXPOSED ########
-
+def sanitize_object_type(object_type):
-
+    if not is_valid_object_type(object_type):
+        return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
 ########  ########

bin/lib/Domain.py

@@ -292,7 +292,7 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F
                 if item_screenshot:
                     dict_item['screenshot'] = Item.get_item_screenshot(item)
                 if item_tag:
-                    dict_item['tags'] = Tag.get_item_tags_minimal(item)
+                    dict_item['tags'] = Tag.get_obj_tags_minimal(item)
                 item_crawled['items'].append(dict_item)
     return item_crawled
 
@@ -365,7 +365,7 @@ def get_domain_tags(domain):
 
     :param domain: crawled domain
     '''
-    return Tag.get_item_tags(domain)
+    return Tag.get_obj_tag(domain)
 
 def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False):
     '''

bin/lib/Screenshot.py

@@ -43,13 +43,16 @@ def get_screenshot_items_list(sha256_string):
     else:
         return []
 
+def get_item_screenshot(item_id):
+    return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
+
 def get_item_screenshot_list(item_id):
     '''
     Retun all decoded item of a given item id.
 
     :param item_id: item id
     '''
-    screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
+    screenshot = get_item_screenshot(item_id)
     if screenshot:
         return [screenshot]
     else:

bin/packages/Date.py

@@ -79,6 +79,9 @@ class Date(object):
         comp_day = str(computed_date.day).zfill(2)
         return comp_year + comp_month + comp_day
 
+def get_today_date_str():
+    return datetime.date.today().strftime("%Y%m%d")
+
 def date_add_day(date, num_day=1):
     new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
     new_date = str(new_date).replace('-', '')

bin/packages/Item.py

@@ -104,7 +104,7 @@ def get_item(request_dict):
         dict_item['date'] = get_item_date(item_id, add_separator=add_separator)
     tags = request_dict.get('tags', True)
     if tags:
-        dict_item['tags'] = Tag.get_item_tags(item_id)
+        dict_item['tags'] = Tag.get_obj_tag(item_id)
 
     size = request_dict.get('size', False)
     if size:
@@ -242,7 +242,7 @@ def get_item_pgp_correlation(item_id):
 def get_item_list_desc(list_item_id):
     desc_list = []
     for item_id in list_item_id:
-        desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_item_tags(item_id)} )
+        desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} )
     return desc_list
 
 # # TODO: add an option to check the tag

bin/packages/Tag.py

@@ -4,13 +4,15 @@
 import os
 import sys
 import redis
+import datetime
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Date
 import Item
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
-import Domain
+import Correlate_object
 
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
@@ -35,6 +37,19 @@ def build_unsafe_tags():
 # set of unsafe tags
 unsafe_tags = build_unsafe_tags()
 
+def is_tags_safe(ltags):
+    '''
+    Check if a list of tags contain an unsafe tag (CE, ...)
+
+    :param ltags: list of tags
+    :type ltags: list
+    :return: is a tag in the unsafe set
+    :rtype: boolean
+    '''
+    return unsafe_tags.isdisjoint(ltags)
+
+#### Taxonomies - Galaxies ####
+
 def get_taxonomie_from_tag(tag):
     return tag.split(':')[0]
 
@@ -49,6 +64,12 @@ def get_active_taxonomies():
 def get_active_galaxies():
     return r_serv_tags.smembers('active_galaxies')
 
+def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update
+    return r_serv_tags.smembers('active_taxonomies_tags')
+
+def get_all_galaxies_tags(): # # TODO: add + REMOVE + Update
+    return r_serv_tags.smembers('active_galaxies_tags')
+
 def is_taxonomie_tag_enabled(taxonomie, tag):
     if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
         return True
@@ -79,6 +100,7 @@ def enable_taxonomy(taxonomie, enable_tags=True):
             # activate taxonomie tags
             for tag in taxonomie_info.machinetags():
                 r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
+                #r_serv_tags.sadd('active_taxonomies_tags', tag)
         else:
             print('Error: {}, please update pytaxonomies'.format(taxonomie))
 
@@ -105,21 +127,7 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
                 return False
     return True
 
-def get_tag_metadata(tag):
+####  ####
-    first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
-    last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen}
-
-def is_tags_safe(ltags):
-    '''
-    Check if a list of tags contain an unsafe tag (CE, ...)
-
-    :param ltags: list of tags
-    :type ltags: list
-    :return: is a tag in the unsafe set
-    :rtype: boolean
-    '''
-    return unsafe_tags.isdisjoint(ltags)
 
 def is_tag_in_all_tag(tag):
     if r_serv_tags.sismember('list_tags', tag):
@@ -127,20 +135,6 @@ def is_tag_in_all_tag(tag):
     else:
         return False
 
-def get_all_tags():
-    return list(r_serv_tags.smembers('list_tags'))
-
-def get_item_tags(item_id):
-    '''
-    Retun all the tags of a given item.
-    :param item_id: (Paste or domain)
-    '''
-    tags = r_serv_metadata.smembers('tag:{}'.format(item_id))
-    if tags:
-        return list(tags)
-    else:
-        return []
-
 def get_min_tag(tag):
     tag = tag.split('=')
     if len(tag) > 1:
@@ -154,8 +148,8 @@ def get_min_tag(tag):
         tag = tag[0]
     return tag
 
-def get_item_tags_minimal(item_id):
+def get_obj_tags_minimal(item_id):
-    return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
+    return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_obj_tag(item_id) ]
 
 def unpack_str_tags_list(str_tags_list):
     str_tags_list = str_tags_list.replace('"','\"')
@@ -164,159 +158,92 @@ def unpack_str_tags_list(str_tags_list):
     else:
         return []
 
+# used by modal
+def get_modal_add_tags(item_id, object_type='item'):
+    '''
+    Modal: add tags to domain or Paste
+    '''
+    return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
+            "object_id": item_id, "object_type": object_type}
 
-# TEMPLATE + API QUERY
+######## NEW VERSION ########
-def add_items_tag(tags=[], galaxy_tags=[], item_id=None): ## TODO: remove me
+def get_tag_first_seen(tag, r_int=False):
-    res_dict = {}
+    '''
-    if item_id == None:
+    Get tag first seen (current: item only)
-        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
+    '''
-    if not tags and not galaxy_tags:
+    res = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
-        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+    if r_int:
-
+        if res is None:
-    res_dict['tags'] = []
+            return 99999999
-    for tag in tags:
-        taxonomie = get_taxonomie_from_tag(tag)
-        if is_taxonomie_tag_enabled(taxonomie, tag):
-            add_item_tag(tag, item_id)
-            res_dict['tags'].append(tag)
         else:
-            return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
+            return int(res)
+    return res
 
-    for tag in galaxy_tags:
+def get_tag_last_seen(tag, r_int=False):
-        galaxy = get_galaxy_from_tag(tag)
+    '''
-        if is_galaxy_tag_enabled(galaxy, tag):
+    Get tag last seen (current: item only)
-            add_item_tag(tag, item_id)
+    '''
-            res_dict['tags'].append(tag)
+    res = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+    if r_int:
+        if res is None:
+            return 0
         else:
-            return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
+            return int(res)
+    return res
 
-    res_dict['id'] = item_id
+def get_tag_metadata(tag, r_int=False):
-    return (res_dict, 200)
+    '''
+    Get tag metadata (current: item only)
+    '''
+    tag_metadata = {"tag": tag}
+    tag_metadata['first_seen'] = get_tag_first_seen(tag)
+    tag_metadata['last_seen'] = get_tag_last_seen(tag)
+    return tag_metadata
 
-
+def get_tags_min_last_seen(l_tags, r_int=False):
-# TEMPLATE + API QUERY
+    '''
-def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
+    Get max last seen from a list of tags (current: item only)
-    res_dict = {}
+    '''
-    if item_id == None:
+    min_last_seen = 99999999
-        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
+    for tag in l_tags:
-    if not tags and not galaxy_tags:
+        last_seen = get_tag_last_seen(tag, r_int=True)
-        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+        if last_seen < min_last_seen:
-    if item_type not in ('paste', 'domain'):
+                min_last_seen = last_seen
-        return ({'status': 'error', 'reason': 'Incorrect item_type'}, 400)
+    if r_int:
-
+        return min_last_seen
-    res_dict['tags'] = []
-    for tag in tags:
-        if tag:
-            taxonomie = get_taxonomie_from_tag(tag)
-            if is_taxonomie_tag_enabled(taxonomie, tag):
-                add_item_tag(tag, item_id, item_type=item_type)
-                res_dict['tags'].append(tag)
-            else:
-                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
-
-    for tag in galaxy_tags:
-        if tag:
-            galaxy = get_galaxy_from_tag(tag)
-            if is_galaxy_tag_enabled(galaxy, tag):
-                add_item_tag(tag, item_id, item_type=item_type)
-                res_dict['tags'].append(tag)
-            else:
-                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
-
-    res_dict['id'] = item_id
-    res_dict['type'] = item_type
-    return (res_dict, 200)
-
-def add_domain_tag(tag, domain, item_date):
-    r_serv_tags.sadd('list_tags:domain', tag)
-    r_serv_metadata.sadd('tag:{}'.format(domain), tag)
-    r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
-
-def add_item_tag(tag, item_path, item_type="paste", tag_date=None):
-
-    if item_type=="paste":
-        item_date = int(Item.get_item_date(item_path))
-
-        #add tag
-        r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
-        r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
-
-        if Item.is_crawled(item_path):
-            domain = Item.get_item_domain(item_path)
-            r_serv_metadata.sadd('tag:{}'.format(domain), tag)
-            r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
-    # domain item
     else:
-        item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
+        return str(min_last_seen)
-        add_domain_tag(tag, item_path, item_date)
 
-    r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
+def is_obj_tagged(object_id, tag):
+    '''
+    Check if a object is tagged
 
-    tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+    :param object_id: object id
-    if tag_first_seen is None:
+    :type domain: str
-        tag_first_seen = 99999999
+    :param tag: object type
+    :type domain: str
+
+    :return: is object tagged
+    :rtype: boolean
+    '''
+    return r_serv_metadata.sismember('tag:{}'.format(object_id), tag)
+
+def get_all_tags():
+    return list(r_serv_tags.smembers('list_tags'))
+
+def get_all_obj_tags(object_type):
+    return list(r_serv_tags.smembers('list_tags:{}'.format(object_type)))
+
+def get_obj_tag(object_id):
+    '''
+    Retun all the tags of a given object.
+    :param object_id: (item_id, domain, ...)
+    '''
+    res = r_serv_metadata.smembers('tag:{}'.format(object_id))
+    if res:
+        return list(res)
     else:
-        tag_first_seen = int(tag_first_seen)
+        return []
-    tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if tag_last_seen is None:
-        tag_last_seen = 0
-    else:
-        tag_last_seen = int(tag_last_seen)
-
-    #add new tag in list of all used tags
-    r_serv_tags.sadd('list_tags', tag)
-
-    # update fisrt_seen/last_seen
-    if item_date < tag_first_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
-
-    # update metadata last_seen
-    if item_date > tag_last_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
-
-# API QUERY
-def remove_item_tags(tags=[], item_id=None):
-    if item_id == None:
-        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
-    if not tags:
-        return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
-
-    dict_res = {}
-    dict_res['tags'] = []
-    for tag in tags:
-        res = remove_item_tag(tag, item_id)
-        if res[1] != 200:
-            return res
-        else:
-            dict_res['tags'].append(tag)
-    dict_res['id'] = item_id
-    return (dict_res, 200)
-
-# TEMPLATE + API QUERY
-def remove_item_tag(tag, item_id):
-    item_date = int(Item.get_item_date(item_id))
-
-    #remove tag
-    r_serv_metadata.srem('tag:{}'.format(item_id), tag)
-    res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_id)
-
-    if res ==1:
-        # no tag for this day
-        if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
-            r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
-        else:
-            r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
-
-        tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
-        tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
-        # update fisrt_seen/last_seen
-        if item_date == tag_first_seen:
-            update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
-        if item_date == tag_last_seen:
-            update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
-        return ({'status': 'success'}, 200)
-    else:
-        return ({'status': 'error', 'reason': 'Item id or tag not found'}, 400)
 
 def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
     if tag_first_seen == tag_last_seen:
@@ -324,7 +251,6 @@ def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
             r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
         # no tag in db
         else:
-            r_serv_tags.srem('list_tags', tag)
             r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
             r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
     else:
@@ -340,7 +266,6 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
             r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
         # no tag in db
         else:
-            r_serv_tags.srem('list_tags', tag)
             r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
             r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
     else:
@@ -350,11 +275,285 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
             tag_last_seen = Date.date_substract_day(tag_last_seen)
             update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
 
+def update_tag_metadata(tag, tag_date, object_type=None, add_tag=True):
+    '''
+    Update tag metadata (current: item only)
+    '''
+    if object_type=="item":
+        # get object metadata
+        tag_metadata = get_tag_metadata(tag, r_int=True)
+        #############
+        ## ADD tag ##
+        if add_tag:
+            # update fisrt_seen
+            if tag_date < tag_metadata['first_seen']:
+                r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_date)
+            # update last_seen
+            if tag_date > tag_metadata['last_seen']:
+                r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_date)
+        ################
+        ## REMOVE tag ##
+        else:
+            if tag_date == tag_metadata['first_seen']:
+                update_tag_first_seen(object_type, tag, tag_metadata['first_seen'], tag_metadata['last_seen'])
+            if tag_date == tag_metadata['last_seen']:
+                update_tag_last_seen(tag, tag_metadata['first_seen'], tag_metadata['last_seen'])
 
-# used by modal
+def update_tag_global_by_obj_type(object_type, tag):
-def get_modal_add_tags(item_id, tag_type='paste'):
+    tag_deleted = False
+    if object_type=='item':
+        if not r_serv_tags.exists('tag_metadata:{}'.format(tag)):
+            tag_deleted = True
+    else:
+        if not r_serv_tags.exists('{}:{}'.format(object_type, tag)):
+            tag_deleted = True
+    if tag_deleted:
+        # update object global tags
+        r_serv_tags.srem('list_tags:{}'.format(object_type), tag)
+        # update global tags
+        for obj_type in Correlate_object.get_all_objects():
+            if r_serv_tags.exists('{}:{}'.format(obj_type, tag)):
+                tag_deleted = False
+        if tag_deleted:
+            r_serv_tags.srem('list_tags', tag)
+
+def add_global_tag(tag, object_type=None):
     '''
-    Modal: add tags to domain or Paste
+    Create a set of all tags used in AIL (all + by object)
+
+    :param tag: tag
+    :type domain: str
+    :param object_type: object type
+    :type domain: str
     '''
-    return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
+    r_serv_tags.sadd('list_tags', tag)
-            "item_id": item_id, "type": tag_type}
+    if object_type:
+        r_serv_tags.sadd('list_tags:{}'.format(object_type), tag)
+
+def add_obj_tags(object_id, object_type, tags=[], galaxy_tags=[]):
+    obj_date = get_obj_date(object_type, object_id)
+    for tag in tags:
+        if tag:
+            taxonomie = get_taxonomie_from_tag(tag)
+            if is_taxonomie_tag_enabled(taxonomie, tag):
+                add_tag(object_type, tag, object_id, obj_date=obj_date)
+            else:
+                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
+
+    for tag in galaxy_tags:
+        if tag:
+            galaxy = get_galaxy_from_tag(tag)
+            if is_galaxy_tag_enabled(galaxy, tag):
+                add_tag(object_type, tag, object_id, obj_date=obj_date)
+            else:
+                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
+
+# TEMPLATE + API QUERY
+def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"):
+    res_dict = {}
+    if object_id == None:
+        return ({'status': 'error', 'reason': 'object_id id not found'}, 404)
+    if not tags and not galaxy_tags:
+        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+    if object_type not in ('item', 'domain'):  # # TODO: put me in another file
+        return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
+
+    # remove empty tags
+    tags = list(filter(bool, tags))
+    galaxy_tags = list(filter(bool, galaxy_tags))
+
+    res = add_obj_tags(object_id, object_type, tags=tags, galaxy_tags=galaxy_tags)
+    if res:
+        return res
+
+    res_dict['tags'] = tags + galaxy_tags
+    res_dict['id'] = object_id
+    res_dict['type'] = object_type
+    return (res_dict, 200)
+
+def add_obj_tag(object_type, object_id, tag, obj_date=None):
+    if object_type=="item": # # TODO: # FIXME: # REVIEW: rename me !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        if obj_date is None:
+            raise ValueError("obj_date is None")
+
+        # add tag
+        r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
+        r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id)
+
+        # add domain tag
+        if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
+            domain = Item.get_item_domain(object_id)
+            add_tag("domain", tag, domain)
+    else:
+        r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
+        r_serv_tags.sadd('{}:{}'.format(object_type, tag), object_id)
+
+def add_tag(object_type, tag, object_id, obj_date=None):
+    # new tag
+    if not is_obj_tagged(object_id, tag):
+        # # TODO: # FIXME: sanityze object_type
+        if not obj_date:
+            obj_date = get_obj_date(object_type, object_id)
+        add_global_tag(tag, object_type=object_type)
+        add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
+        update_tag_metadata(tag, obj_date)
+
+    # create tags stats  # # TODO:  put me in cache
+    r_serv_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
+
+def delete_obj_tag(object_type, object_id, tag, obj_date):
+    if object_type=="item": # # TODO: # FIXME: # REVIEW: rename me !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        obj_date = get_obj_date(object_type, object_id)
+        r_serv_metadata.srem('tag:{}'.format(object_id), tag)
+        r_serv_tags.srem('{}:{}'.format(tag, obj_date), object_id)
+    else:
+        r_serv_metadata.srem('tag:{}'.format(object_id), tag)
+        r_serv_tags.srem('{}:{}'.format(object_type, tag), object_id)
+
+def delete_tag(object_type, tag, object_id, obj_date=None):
+    # tag exist
+    if is_obj_tagged(object_id, tag):
+        if not obj_date:
+            obj_date = get_obj_date(object_type, object_id)
+        delete_obj_tag(object_type, object_id, tag, obj_date)
+        update_tag_metadata(tag, obj_date, object_type=object_type, add_tag=False)
+        update_tag_global_by_obj_type(object_type, tag)
+
+    else:
+        return ({'status': 'error', 'reason': 'object id or tag not found', 'value': tag}, 400)
+
+# API QUERY
+def api_delete_obj_tags(tags=[], object_id=None, object_type="item"):
+    if not object_id:
+        return ({'status': 'error', 'reason': 'object id not found'}, 404)
+    if not tags:
+        return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
+
+    res = delete_obj_tags(object_id, object_type, tags=tags)
+    if res:
+        return res
+
+    dict_res = {}
+    dict_res['tags'] = tags
+    dict_res['id'] = object_id
+    return (dict_res, 200)
+
+def delete_obj_tags(object_id, object_type, tags=[]):
+    obj_date = get_obj_date(object_type, object_id)
+    for tag in tags:
+        res = delete_tag(object_type, tag, object_id, obj_date=obj_date)
+        if res:
+            return res
+
+def sanitise_tags_date_range(l_tags, date_from=None, date_to=None):
+    if date_from and date_to is None:
+        date_from = get_tags_min_last_seen(l_tags, r_int=False)
+        date_to = date_from
+    return Date.sanitise_date_range(date_from, date_to)
+
+
+# # TODO: verify tags + object_type
+# get set_keys: intersection
+def get_obj_keys_by_tags(object_type, l_tags, date_day=None):
+    l_set_keys = []
+    if object_type=='item':
+        for tag in l_tags:
+            l_set_keys.append('{}:{}'.format(tag, date_day))
+    else:
+        for tag in l_tags:
+            l_set_keys.append('{}:{}'.format(object_type, tag))
+    return l_set_keys
+
+def get_obj_by_tag(key_tag):
+    return r_serv_tags.smembers(key_tag)
+
+def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object
+    # with daterange
+    l_tagged_obj = []
+    if object_type=='item':
+        #sanityze date
+        date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to)
+        l_dates = Date.substract_date(date_from, date_to)
+
+        for date_day in l_dates:
+            l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day)
+            # if len(l_set_keys) > nb_obj:
+            #     return l_tagged_obj
+            if len(l_set_keys) < 2:
+                date_day_obj = get_obj_by_tag(l_set_keys[0])
+            else:
+                date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
+
+            # next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj
+            # if next_nb_start > 0:
+            #  get + filter nb_start
+            l_tagged_obj.extend( date_day_obj )
+
+        # handle pagination
+        nb_all_elem = len(l_tagged_obj)
+        nb_pages = nb_all_elem / nb_obj
+        if not nb_pages.is_integer():
+            nb_pages = int(nb_pages)+1
+        else:
+            nb_pages = int(nb_pages)
+        if page > nb_pages:
+            page = nb_pages
+
+        # select index
+        start = nb_obj*(page -1)
+        stop = (nb_obj*page) -1
+        l_tagged_obj = l_tagged_obj[start:stop]
+
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop+1, "nb_all_elem":nb_all_elem}
+
+    # without daterange
+    else:
+        l_set_keys = get_obj_keys_by_tags(object_type, l_tags)
+        if len(l_set_keys) < 2:
+            l_tagged_obj = get_obj_by_tag(l_set_keys[0])
+        else:
+            l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
+
+        if not l_tagged_obj:
+            return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
+
+        # handle pagination
+        nb_all_elem = len(l_tagged_obj)
+        nb_pages = nb_all_elem / nb_obj
+        if not nb_pages.is_integer():
+            nb_pages = int(nb_pages)+1
+        else:
+            nb_pages = int(nb_pages)
+        if page > nb_pages:
+            page = nb_pages
+
+        # multiple pages
+        if nb_pages > 1:
+            start = nb_obj*(page -1)
+            stop = (nb_obj*page) -1
+            current_index = 0
+            l_obj = []
+            for elem in l_tagged_obj:
+                if current_index > stop:
+                    break
+                if start <= current_index and stop >= current_index:
+                    l_obj.append(elem)
+                current_index += 1
+            l_tagged_obj = l_obj
+            stop += 1
+            if stop > nb_all_elem:
+                stop = nb_all_elem
+        # only one page
+        else:
+            start = 0
+            stop = nb_all_elem
+            l_tagged_obj = list(l_tagged_obj)
+
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
+
+
+def get_obj_date(object_type, object_id): # # TODO: move me in another file
+    if object_type == "item":
+        return Item.get_item_date(object_id)
+    else:
+        return None

bin/submit_paste.py

@@ -16,6 +16,9 @@ import sflock
 from Helper import Process
 from pubsublogger import publisher
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Tag
+
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
 
@@ -50,10 +53,10 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
 
     # add tags
     for tag in ltags:
-        add_item_tag(tag, rel_item_path)
+        Tag.add_tag('item', tag, rel_item_path)
 
     for tag in ltagsgalaxies:
-        add_item_tag(tag, rel_item_path)
+        Tag.add_tag('item', tag, rel_item_path)
 
     r_serv_log_submit.incr(uuid + ':nb_end')
     r_serv_log_submit.incr(uuid + ':nb_sucess')
@@ -108,37 +111,6 @@ def get_item_date(item_filename):
     l_directory = item_filename.split('/')
     return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
 
-def add_item_tag(tag, item_path):
-    item_date = int(get_item_date(item_path))
-
-    #add tag
-    r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
-    r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
-
-    r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
-
-    tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if tag_first_seen is None:
-        tag_first_seen = 99999999
-    else:
-        tag_first_seen = int(tag_first_seen)
-    tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if tag_last_seen is None:
-        tag_last_seen = 0
-    else:
-        tag_last_seen = int(tag_last_seen)
-
-    #add new tag in list of all used tags
-    r_serv_tags.sadd('list_tags', tag)
-
-    # update fisrt_seen/last_seen
-    if item_date < tag_first_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
-
-    # update metadata last_seen
-    if item_date > tag_last_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
-
 def verify_extention_filename(filename):
     if not '.' in filename:
         return True

bin/update-background.py

@@ -86,11 +86,31 @@ if __name__ == "__main__":
         update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py')
         process = subprocess.run(['python' ,update_file])
 
-        update_progress = r_serv_db.get('ail:current_background_script_stat')
+        update_progress = r_serv.get('ail:current_background_script_stat')
         if update_progress:
             if int(update_progress) == 100:
                 r_serv.delete('ail:update_in_progress')
                 r_serv.delete('ail:current_background_script')
                 r_serv.delete('ail:current_background_script_stat')
                 r_serv.delete('ail:current_background_update')
-                r_serv_db.srem('ail:to_update', new_version)
+                r_serv.srem('ail:to_update', new_version)
+
+    elif r_serv.sismember('ail:to_update', 'v2.7'):
+        new_version = 'v2.7'
+        r_serv.delete('ail:update_error')
+        r_serv.delete('ail:current_background_script_stat')
+        r_serv.set('ail:update_in_progress', new_version)
+        r_serv.set('ail:current_background_update', new_version)
+        r_serv.set('ail:current_background_script', 'domain tags update')
+
+        update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py')
+        process = subprocess.run(['python' ,update_file])
+
+        update_progress = r_serv.get('ail:current_background_script_stat')
+        if update_progress:
+            if int(update_progress) == 100:
+                r_serv.delete('ail:update_in_progress')
+                r_serv.delete('ail:current_background_script')
+                r_serv.delete('ail:current_background_script_stat')
+                r_serv.delete('ail:current_background_update')
+                r_serv.srem('ail:to_update', new_version)

tests/testApi.py

@@ -128,7 +128,7 @@ class TestApiV1(unittest.TestCase):
     # POST api/v1/add/item/tag
     def test_0007_api_add_item_tag(self):
         tags_to_add = ["infoleak:analyst-detection=\"api-key\""]
-        current_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
         current_item_tag.append(tags_to_add[0])
 
         #galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""]
@@ -138,7 +138,7 @@ class TestApiV1(unittest.TestCase):
         item_tags = req_json['tags']
         self.assertEqual(item_tags, tags_to_add)
 
-        new_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        new_item_tag = Tag.get_obj_tag(self.__class__.item_id)
         self.assertCountEqual(new_item_tag, current_item_tag)
 
     # DELETE api/v1/delete/item/tag
@@ -149,7 +149,7 @@ class TestApiV1(unittest.TestCase):
         req_json = parse_response(self, req)
         item_tags = req_json['tags']
         self.assertCountEqual(item_tags, tags_to_delete)
-        current_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
         if tags_to_delete[0] in current_item_tag:
             self.fail('Tag no deleted')
 

update/v1.5/Update-ARDB_Tags.py

@@ -65,6 +65,9 @@ if __name__ == '__main__':
                 tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
             nb_tags_to_update += r_serv_tag.scard(tag)
 
+        if nb_tags_to_update == 0:
+            nb_tags_to_update = 1
+
         for tag in tags_list:
 
             all_item =  r_serv_tag.smembers(tag)

update/v2.4/Update_domain.py

@@ -28,9 +28,9 @@ def update_update_stats():
 def update_domain_by_item(domain_obj, item_id):
     domain_name = domain_obj.get_domain_name()
     # update domain tags
-    for tag in Tag.get_item_tags(item_id):
+    for tag in Tag.get_obj_tag(item_id):
         if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
-            Tag.add_domain_tag(tag, domain_name, Item.get_item_date(item_id))
+            Tag.add_tag("domain", tag, domain_name, obj_date=Item.get_item_date(item_id))
 
     # update domain correlation
     item_correlation = Item.get_item_all_correlation(item_id)

update/v2.7/Update.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v2.7'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
+    r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
+    config_loader = None
+
+    #Set current update_in_progress
+    r_serv.set('ail:update_in_progress', new_version)
+    r_serv.set('ail:current_background_update', new_version)
+
+    r_serv.sadd('ail:to_update', new_version)
+
+    #### Update tags ####
+    r_serv_tags.sunionstore('list_tags:item', 'list_tags', [])
+    r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', [])
+    r_serv_onion.delete('incorrect_domain')
+    r_serv.set('ail:update_v2.7:deletetagrange', 1)
+    ####     ####
+
+    #Set current ail version
+    r_serv.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v2.7/Update.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v2.7/Update.py
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

update/v2.7/Update_domain_tags.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import datetime
+
+from hashlib import sha256
+
+from pyfaup.faup import Faup
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Date
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+import Tag
+
+def sanitize_domain(domain):
+    faup.decode(domain)
+    domain_sanitized = faup.get()
+    domain_sanitized = domain_sanitized['domain']
+    try:
+        domain_sanitized = domain_sanitized.decode()
+    except:
+        pass
+    return domain_sanitized.lower()
+
+def delete_domain_tag_daterange():
+    all_domains_tags = Tag.get_all_obj_tags('domain')
+    nb_updated = 0
+    nb_to_update = len(all_domains_tags)
+    if nb_to_update == 0:
+        nb_to_update = 1
+    refresh_time = time.time()
+    l_dates = Date.substract_date('20191008', Date.get_today_date_str())
+    for tag in all_domains_tags:
+        for date_day in l_dates:
+            r_serv_tags.delete('domain:{}:{}'.format(tag, date_day))
+        nb_updated += 1
+        refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
+    r_serv_db.delete('ail:update_v2.7:deletetagrange')
+
+def update_domain_tags(domain):
+    domain_sanitized = sanitize_domain(domain)
+    if domain != domain_sanitized:
+        r_serv_onion.sadd('incorrect_domain', domain)
+        domain = domain_sanitized
+
+    domain_tags = Tag.get_obj_tag(domain)
+    for tag in domain_tags:
+        # delete incorrect tags
+        if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"':
+            r_serv_metadata.srem('tag:{}'.format(domain), tag)
+        else:
+            Tag.add_global_tag(tag, object_type='domain')
+            r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
+
+def update_progress(refresh_time, nb_updated, nb_elem_to_update):
+    if time.time() - refresh_time > 10:
+        progress = int((nb_updated * 100) / nb_elem_to_update)
+        print('{}/{}    updated    {}%'.format(nb_updated, nb_elem_to_update, progress))
+        r_serv_db.set('ail:current_background_script_stat', progress)
+        refresh_time = time.time()
+
+    return refresh_time
+
+def update_db():
+    nb_updated = 0
+    nb_to_update = r_serv_onion.scard('domain_update_v2.7')
+    refresh_time = time.time()
+    r_serv_db.set('ail:current_background_script_stat', 0)
+    r_serv_db.set('ail:current_background_script', 'domain tags update')
+    domain = r_serv_onion.spop('domain_update_v2.7')
+    while domain is not None:
+        update_domain_tags(domain)
+        nb_updated += 1
+        refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
+        domain = r_serv_onion.spop('domain_update_v2.7')
+    if r_serv_db.exists('ail:update_v2.7:deletetagrange'):
+        r_serv_db.set('ail:current_background_script_stat', 0)
+        r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys')
+        delete_domain_tag_daterange()
+
+    # sort all crawled domain
+    r_serv_onion.sort('full_onion_up', alpha=True)
+    r_serv_onion.sort('full_regular_up', alpha=True)
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+    faup = Faup()
+
+    config_loader = ConfigLoader.ConfigLoader()
+
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
+    r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
+    r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+    config_loader = None
+
+    update_version = 'v2.7'
+
+    r_serv_db.set('ail:update_in_progress', update_version)
+    r_serv_db.set('ail:current_background_update', update_version)
+
+    r_serv_db.set('ail:current_background_script_stat', 0)
+    r_serv_db.set('ail:current_background_script', 'tags update')
+
+    update_db()
+
+    r_serv_db.set('ail:current_background_script_stat', 100)
+
+
+    end = time.time()
+    print('ALL domains tags updated in {} s'.format(end - start_deb))
+
+    r_serv_db.delete('ail:update_in_progress')
+    r_serv_db.delete('ail:current_background_script')
+    r_serv_db.delete('ail:current_background_script_stat')
+    r_serv_db.delete('ail:current_background_update')
+    r_serv_db.srem('ail:to_update', update_version)

var/www/Flask_server.py

@@ -39,6 +39,8 @@ import Flask_config
 from blueprints.root import root
 from blueprints.crawler_splash import crawler_splash
 from blueprints.correlation import correlation
+from blueprints.tags_ui import tags_ui
+
 
 Flask_dir = os.environ['AIL_FLASK']
 
@@ -85,6 +87,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
 app.register_blueprint(root, url_prefix=baseUrl)
 app.register_blueprint(crawler_splash, url_prefix=baseUrl)
 app.register_blueprint(correlation, url_prefix=baseUrl)
+app.register_blueprint(tags_ui, url_prefix=baseUrl)
 # =========       =========#
 
 # ========= session ========

var/www/blueprints/crawler_splash.py

@@ -45,13 +45,18 @@ def api_validator(api_response):
 
 # ============= ROUTES ==============
 # add route : /crawlers/show_domain
-@crawler_splash.route('/crawlers/showDomain')
+@crawler_splash.route('/crawlers/showDomain', methods=['GET', 'POST'])
 @login_required
 @login_read_only
 def showDomain():
-    domain_name = request.args.get('domain')
+    if request.method == 'POST':
-    epoch = request.args.get('epoch')
+        domain_name = request.form.get('in_show_domain')
-    port = request.args.get('port')
+        epoch = None
+        port = None
+    else:
+        domain_name = request.args.get('domain')
+        epoch = request.args.get('epoch')
+        port = request.args.get('port')
 
     res = api_validator(Domain.api_verify_if_domain_exist(domain_name))
     if res:
@@ -73,4 +78,4 @@ def showDomain():
             dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
 
     return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
-                                modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
+                                modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], object_type="domain"))

var/www/blueprints/tags_ui.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+import json
+import random
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required, current_user, login_user, logout_user
+
+sys.path.append('modules')
+import Flask_config
+
+# Import Role_Manager
+from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Date
+import Tag
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import Correlate_object
+
+r_cache = Flask_config.r_cache
+r_serv_db = Flask_config.r_serv_db
+r_serv_tags = Flask_config.r_serv_tags
+bootstrap_label = Flask_config.bootstrap_label
+
+# ============ BLUEPRINT ============
+tags_ui = Blueprint('tags_ui', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/tags'))
+
+# ============ VARIABLES ============
+
+
+
+# ============ FUNCTIONS ============
+
+
+# ============= ROUTES ==============
+@tags_ui.route('/tag/add_tags')
+@login_required
+@login_analyst
+def add_tags():
+
+    tags = request.args.get('tags')
+    tagsgalaxies = request.args.get('tagsgalaxies')
+    object_id = request.args.get('object_id')
+    object_type = request.args.get('object_type')
+
+    list_tag = tags.split(',')
+    list_tag_galaxies = tagsgalaxies.split(',')
+
+    res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, object_id=object_id, object_type=object_type)
+    # error
+    if res[1] != 200:
+        return str(res[0])
+
+    return redirect(Correlate_object.get_item_url(object_type, object_id))
+
+@tags_ui.route('/tag/delete_tag')
+@login_required
+@login_analyst
+def delete_tag():
+
+    object_type = request.args.get('object_type')
+    object_id = request.args.get('object_id')
+    tag = request.args.get('tag')
+
+    res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type)
+    if res[1] != 200:
+        return str(res[0])
+    return redirect(Correlate_object.get_item_url(object_type, object_id))
+
+
+@tags_ui.route('/tag/get_all_tags')
+@login_required
+@login_read_only
+def get_all_tags():
+    return jsonify(Tag.get_all_tags())
+
+@tags_ui.route('/tag/get_all_obj_tags')
+@login_required
+@login_read_only
+def get_all_obj_tags():
+    object_type = request.args.get('object_type')
+    res = Correlate_object.sanitize_object_type(object_type)
+    if res:
+        return jsonify(res)
+    return jsonify(Tag.get_all_obj_tags(object_type))
+
+@tags_ui.route('/tag/search/domain')
+@login_required
+@login_read_only
+def tags_search_domains():
+    object_type = 'domain'
+    dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
+@tags_ui.route('/tag/search/image')
+@login_required
+@login_read_only
+def tags_search_images():
+    object_type = 'image'
+    dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
+@tags_ui.route('/tag/search/get_obj_by_tags')
+@login_required
+@login_read_only
+def get_obj_by_tags():
+
+    # # TODO: sanityze all
+    object_type = request.args.get('object_type')
+    ltags = request.args.get('ltags')
+    page = request.args.get('page')
+    date_from = request.args.get('ltags')
+    date_to = request.args.get('ltags')
+
+    # unpack tags
+    list_tags = ltags.split(',')
+    list_tag = []
+    for tag in list_tags:
+        list_tag.append(tag.replace('"','\"'))
+
+    # object_type
+    res = Correlate_object.sanitize_object_type(object_type)
+    if res:
+        return jsonify(res)
+
+    # page
+    try:
+        page = int(page)
+    except:
+        page = 1
+
+    dict_obj = Tag.get_obj_by_tags(object_type, list_tag, page=page)
+
+    if dict_obj['tagged_obj']:
+        dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s",
+                        "tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'],
+                        "nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']}
+        for obj_id in dict_obj['tagged_obj']:
+            obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
+            obj_metadata['id'] = obj_id
+            dict_tagged["tagged_obj"].append(obj_metadata)
+
+        dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
+
+        if len(list_tag) == 1:
+            dict_tagged['current_tags'] = [ltags.replace('"', '\"')]
+        else:
+            dict_tagged['current_tags'] = list_tag
+        dict_tagged['current_tags_str'] = ltags
+
+        #return jsonify(dict_tagged)
+    else:
+        dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)

var/www/modules/Flask_config.py

@@ -90,7 +90,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
                            'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
                                                               'update_warning_message_notice_me': 'missing from the UI.'},
                            'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
-                                                           'update_warning_message_notice_me': 'missing from the UI.'}
+                                                           'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
+                                                            'update_warning_message_notice_me': 'missing from the UI.'}
                         }
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')

var/www/modules/Tags/Flask_Tags.py

@@ -369,15 +369,14 @@ def get_tags_galaxy():
 @Tags.route("/Tags/remove_tag")
 @login_required
 @login_analyst
-def remove_tag():
+def remove_tag(): #TODO remove me , used by showpaste 
 
-    #TODO verify input
     path = request.args.get('paste')
     tag = request.args.get('tag')
 
-    res = Tag.remove_item_tag(tag, path)
+    res = Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
     if res[1] != 200:
-        str(res[0])
+        return str(res[0])
     return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
 @Tags.route("/Tags/confirm_tag")
@@ -390,11 +389,11 @@ def confirm_tag():
     tag = request.args.get('tag')
 
     if(tag[9:28] == 'automatic-detection'):
-        Tag.remove_item_tag(tag, path)
+        Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
 
         tag = tag.replace('automatic-detection','analyst-detection', 1)
         #add analyst tag
-        Tag.add_item_tag(tag, path)
+        Tag.add_tag('item', tag, path)
 
         return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
@@ -422,49 +421,6 @@ def tag_validation():
     else:
         return 'input error'
 
-@Tags.route("/Tags/addTags")
-@login_required
-@login_analyst
-def addTags():
-
-    tags = request.args.get('tags')
-    tagsgalaxies = request.args.get('tagsgalaxies')
-    path = request.args.get('path')
-
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    res = Tag.add_items_tags(list_tag, list_tag_galaxies, item_id=path)
-    print(res)
-    # error
-    if res[1] != 200:
-        return str(res[0])
-    # success
-    return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
-
-@Tags.route("/Tags/add_item_tags")
-@login_required
-@login_analyst
-def add_item_tags():
-
-    tags = request.args.get('tags')
-    tagsgalaxies = request.args.get('tagsgalaxies')
-    item_id = request.args.get('item_id')
-    item_type = request.args.get('type')
-
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    res = Tag.add_items_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, item_id=item_id, item_type=item_type)
-    # error
-    if res[1] != 200:
-        return str(res[0])
-    # success
-    if item_type=='domain':
-        return redirect(url_for('crawler_splash.showDomain', domain=item_id))
-    else:
-        return redirect(url_for('showsavedpastes.showsavedpaste', paste=item_id))
-
 @Tags.route("/Tags/taxonomies")
 @login_required
 @login_read_only

var/www/modules/Tags/templates/Tags.html

@@ -217,8 +217,8 @@ var last_clicked_paste;
 var can_change_modal_content = true;
 
 $(document).ready(function(){
-	$("#nav_quick_search").removeClass("text-muted");
+	$("#nav_tags_search").removeClass("text-muted");
-	$("#nav_tag_{{tag_nav}}").addClass("active");
+	$("#nav_tags_search_item").addClass("active");
 	search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
 
 	// Use to bind the button with the new displayed data

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -105,6 +105,29 @@
 					</div>
 				</div>
 
+			<div class="d-flex justify-content-center my-4">
+				<div class="card border-secondary" style="max-width: 40rem;">
+					<div class="card-body text-dark">
+						<h5 class="card-title">Show Domain:</h5>
+						<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
+							<div class="input-group mb-3">
+							  <input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
+							  <div class="input-group-append">
+							    <button class="btn btn-info" type="submit" id="btn_show_domain">
+										<i class="fas fa-search"></i>
+									</button>
+							  </div>
+							</div>
+						</form>
+					</div>
+				</div>
+			</div>
+
+
+				{% with object_type='domain' %}
+					{% include 'tags/block_obj_tags_search.html' %}
+				{% endwith %}
+
 			</div>
 		</div>
 	</div>

var/www/modules/restApi/Flask_restApi.py

@@ -246,11 +246,11 @@ def add_item_tags():
     if not data:
         return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
 
-    item_id = data.get('id', None)
+    object_id = data.get('id', None)
     tags = data.get('tags', [])
     galaxy = data.get('galaxy', [])
 
-    res = Tag.add_items_tag(tags, galaxy, item_id)
+    res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -275,10 +275,10 @@ def delete_item_tags():
     if not data:
         return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
 
-    item_id = data.get('id', None)
+    object_id = data.get('id', None)
     tags = data.get('tags', [])
 
-    res = Tag.remove_item_tags(tags, item_id)
+    res = Tag.api_delete_obj_tags(tags=tags, object_id=object_id, object_type="item")
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -570,7 +570,7 @@
       var tags = ltags.getValue()
       var tagsgalaxy = ltagsgalaxies.getValue()
       var path = '{{ request.args.get('paste') }}'
-      window.location.replace("{{ url_for('Tags.addTags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&path=" + path);
+      window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id=" + path + "&object_type=item");
     }
 
 </script>

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -79,10 +79,13 @@
 								</table>
 
 								<div>
+                  {% include 'modals/edit_tag.html' %}
 									{% for tag in dict_domain['tags'] %}
-									<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
+  									<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
-										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                    data-tagid="{{ tag }}" data-objtype="domain" data-objid="{{ dict_domain['domain'] }}">
-									</a>
+                      {{ tag }}
+                    </button>
+
 									{%  endfor %}
 									<br>
                   {% include 'modals/add_tags.html' %}

var/www/templates/modals/add_tags.html

@@ -126,6 +126,6 @@ jQuery("#all-tags-galaxies").click(function(e){
 function addTags() {
   var tags = ltags.getValue()
   var tagsgalaxy = ltagsgalaxies.getValue()
-  window.location.replace("{{ url_for('Tags.add_item_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&item_id={{ modal_add_tags['item_id'] }}&type={{ modal_add_tags['type'] }}");
+  window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}");
 }
 </script>

var/www/templates/modals/edit_tag.html

@@ -0,0 +1,49 @@
+<div id="edit_tags_modal" class="modal fade" role="dialog">
+  <div class="modal-dialog modal-lg">
+
+    <div id="add_tags_modal_content" class="modal-content">
+      <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
+        <h4>Edit Tag</h4>
+        <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+          <span aria-hidden="true">&times;</span>
+        </button>
+      </div>
+
+      <div class="modal-body text-center">
+        <h2><span class="badge badge-warning" id="modal_tag_edit_tag_id">Warning</span></h2>
+      </div>
+
+      <div class="modal-footer">
+
+        <!-- <a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=tp" class="btn btn-outline-success mr-0" data-toggle="tooltip" title="Good Detection">
+          <i class="fas fa-thumbs-up"></i>
+        </a>
+        <a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=fp" class="btn btn-outline-danger mr-auto" data-toggle="tooltip" title="Bad Detection">
+          <i class="fas fa-thumbs-down"></i>
+        </a>
+
+        <a href="{{ url_for('Tags.confirm_tag') }}?object_id=rrrrr&tag=eeeeee" class="btn btn-primary">
+          <i class="fas fa-check"></i> Confirm this Tag
+        </a> -->
+
+        <a href="#" class="btn btn-danger" id="modal_tag_edit_delete_tag">
+          <i class="fas fa-trash-alt"></i> Delete this Tag
+        </a>
+
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+// tagid + objtype + objid
+$('#edit_tags_modal').on('show.bs.modal', function (event) {
+    var button = $(event.relatedTarget);
+    var tagid = button.data('tagid')
+    var objtype = button.data('objtype')
+    var objid = button.data('objid')
+    var modal = $(this)
+    modal.find('#modal_tag_edit_tag_id').text(tagid)
+    modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
+  })
+</script>

var/www/templates/pagination.html

@@ -0,0 +1,50 @@
+<div class="my-2">
+	<div class="d-flex justify-content-center border-top">
+		<nav class="mt-4" aria-label="...">
+			<ul class="pagination">
+				<li class="page-item {%if page==1%}disabled{%endif%}">
+					<a class="page-link" href="{{ target_url }}&page={{page-1}}">Previous</a>
+				</li>
+
+				{%if page>3%}
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page=1">1</a></li>
+					<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>
+					<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
+				{%else%}
+				{%if page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-2}}">{{page-2}}</a></li>{%endif%}
+				{%if page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>{%endif%}
+					<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
+				{%endif%}
+
+				{%if nb_page_max-page>3%}
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page+1}}">{{page+1}}</a></li>
+					<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>
+				{%else%}
+				{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-2}}">{{nb_page_max-2}}</a></li>{%endif%}
+				{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-1}}">{{nb_page_max-1}}</a></li>{%endif%}
+				{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>{%endif%}
+				{%endif%}
+
+				<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
+					<a class="page-link" href="{{ target_url }}&page={{page+1}}" aria-disabled="true">Next</a>
+				</li>
+			</ul>
+		</nav>
+	</div>
+
+	{%if nb_all_elem%}
+		<div class="d-flex justify-content-center">
+			<span class="badge badge-info text-white">
+				{{object_name}}:&nbsp;
+				<span class="badge badge-light">{{nb_first_elem}}-{{nb_last_elem}}</span>
+				<span> / </span>
+				<span class="badge badge-light">{{nb_all_elem}}</span>
+			 </span>
+		</div>
+		<br>
+		<br>
+		<br>
+	{%endif%}
+</div>

var/www/templates/tags/block_obj_tags_search.html

@@ -0,0 +1,103 @@
+<div class="card mb-3 mt-1">
+	<div class="card-header text-white bg-dark">
+		<h5 class="card-title">Search {{object_name}} by Tags :</h5>
+	</div>
+		<div class="card-body">
+
+			<!-- <div class="row mb-3">                      TODO: use condition
+				<div class="col-md-6">
+					<div class="input-group" id="date-range-from">
+						<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+						<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
+					</div>
+				</div>
+				<div class="col-md-6">
+					<div class="input-group" id="date-range-to">
+						<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+						<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
+					</div>
+				</div>
+			</div> -->
+
+			<div class="input-group mb-3">
+			  <div class="input-group-prepend">
+					<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
+						<i class="fas fa-eraser"></i>
+					</button>
+			  </div>
+				<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
+			</div>
+
+			<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
+				<i class="fas fa-search"></i> Search {{object_name}}
+			</button>
+
+			</div>
+	</div>
+
+
+<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+<script>
+  var ltags;
+  $.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{ object_type }}",
+    function(data) {
+      ltags = $('#ltags').tagSuggest({
+        data: data,
+				value: [{%if "current_tags"%}{% for tag in current_tags %}'{{tag|safe}}',{%endfor%}{%endif%}],
+        sortOrder: 'name',
+        maxDropHeight: 200,
+        name: 'ltags'
+      });
+  });
+
+  function searchTags() {
+  	var data = ltags.getValue();
+  	//var date_from = $('#date-range-from-input').val();
+  	//var date_to =$('#date-range-to-input').val();
+		//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
+		var parameter = "?ltags=" + data + "&object_type={{ object_type }}{%if page%}&page={{ page }}{%endif%}";
+  	window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}" + parameter);
+  }
+  function emptyTags() {
+  	ltags.clear();
+  }
+
+	// $('#date-range-from').dateRangePicker({
+	// 			separator : ' to ',
+	// 			getValue: function(){
+	// 					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+	// 							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+	// 					else
+	// 							return '';
+	// 					},
+	// 					setValue: function(s,s1,s2){
+	// 							$('#date-range-from-input').val(s1);
+	// 							$('#date-range-to-input').val(s2);
+	// 					}
+	// });
+	// $('#date-range-to').dateRangePicker({
+	// 		separator : ' to ',
+	// 		getValue: function(){
+	// 				if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+	// 						return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+	// 				else
+	// 						return '';
+	// 		},
+	// 		setValue: function(s,s1,s2){
+	// 				$('#date-range-from-input').val(s1);
+	// 				$('#date-range-to-input').val(s2);
+	// 		}
+	// });
+
+// tagid + objtype + objid
+// $('#edit_tags_modal').on('show.bs.modal', function (event) {
+//     var button = $(event.relatedTarget);
+//     var tagid = button.data('tagid')
+//     var objtype = button.data('objtype')
+//     var objid = button.data('objid')
+//     var modal = $(this)
+//     modal.find('#modal_tag_edit_tag_id').text(tagid)
+//     modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
+//   })
+</script>

var/www/templates/tags/menu_sidebar.html

@@ -6,6 +6,29 @@
       	</button>
 
       	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
+          <h5 class="d-flex text-muted w-100" id="nav_tags_search">
+      		  <span>Tags Search</span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('Tags.Tags_page') }}" id="nav_tags_search_item">
+      	        <i class="far fa-file"></i>
+      	        Search Items by Tags
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_domains') }}" id="nav_tags_search_domain">
+      					<i class="fab fa-html5"></i>
+      	       	Search Domains by Tags
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image">
+      					<i class="fas fa-image"></i>
+      	       	Search Images by Tags
+      	      </a>
+      	    </li>
+          </ul>
       		<h5 class="d-flex text-muted w-100">
       		  <span>Tags Management </span>
       		</h5>

var/www/templates/tags/search_obj_by_tags.html

@@ -0,0 +1,138 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Tags - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+        {% with object_type=dict_tagged['object_type'], current_tags=dict_tagged['current_tags'], object_name=dict_tagged['object_name'] %}
+					{% include 'tags/block_obj_tags_search.html' %}
+				{% endwith %}
+
+				<div>
+					{%if 'tagged_obj' in dict_tagged%}
+						<table class="table table-bordered table-hover" id="myTable_">
+				      <thead class="thead-dark">
+				        <tr>
+                  {%if dict_tagged["object_type"]=="domain"%}
+  				          <th>first seen</th>
+  				          <th>last check</th>
+                    <th style="max-width: 800px;">Domain</th>
+  				          <th>status</th>
+                  {%endif%}
+				        </tr>
+				      </thead>
+				      <tbody>
+
+                {%if dict_tagged["object_type"]=="domain"%}
+
+  				        {% for dict_obj in dict_tagged["tagged_obj"] %}
+  				          <tr>
+                      <td class="pb-0">{{ dict_obj['first_seen'] }}</td>
+                      <td class="pb-0">{{ dict_obj['last_check'] }}</td>
+  										<td class="pb-0">
+                        <a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_obj['id']}}" class="text-secondary">
+                          <div style="line-height:0.9;">{{ dict_obj['id'] }}</div>
+                        </a>
+                        <div class="mb-2">
+  				                {% for tag in dict_obj['tags'] %}
+    				                <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
+    				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+    				                </a>
+  				                {%  endfor %}
+  				              </div>
+                      </td>
+  				            <td class="pb-0">
+                        {%if dict_obj['status'] %}
+      										<div style="color:Green;">
+      											<i class="fas fa-check-circle"></i>
+      											UP
+      										</div>
+      									{% else %}
+      										<div style="color:Red;">
+      											<i class="fas fa-times-circle"></i>
+      											DOWN
+      										</div>
+      									{% endif %}
+  										</td>
+  				          </tr>
+  				        {% endfor %}
+
+                {% endif %}
+
+				      </tbody>
+				    </table>
+
+					{%endif%}
+				</div>
+        {%if 'tagged_obj' in dict_tagged%}
+          {% with page=dict_tagged['page'], nb_page_max=dict_tagged['nb_pages'],  nb_first_elem=dict_tagged['nb_first_elem'], nb_last_elem=dict_tagged['nb_last_elem'], nb_all_elem=dict_tagged['nb_all_elem'] %}
+            {% set object_name= dict_tagged['object_name'] %}
+            {% set target_url=url_for('tags_ui.get_obj_by_tags') + "?object_type=" + dict_tagged['object_type'] + "&ltags=" + dict_tagged['current_tags_str'] %}
+            {% include 'pagination.html' %}
+  				{% endwith %}
+        {%endif%}
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+var ltags;
+var search_table;
+
+$(document).ready(function(){
+  $("#nav_tags_search").removeClass("text-muted");
+	$("#nav_tags_search_{{dict_tagged['object_type']}}").addClass("active");
+  $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 10, "order": [[ 0, "asc" ]] });
+
+});
+</script>
+
+<script>
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</html>