chg: [tags] refactor tags + cleanup

.gitignore

@@ -2,6 +2,7 @@
 *.swp
 *.pyc
 *.swo
+.idea
 
 # Install Dirs
 AILENV

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "files/misp-taxonomies"]
 	path = files/misp-taxonomies
 	url = https://github.com/MISP/misp-taxonomies.git
+[submodule "files/misp-galaxy"]
+	path = files/misp-galaxy
+	url = https://github.com/MISP/misp-galaxy.git

bin/DB_KVROCKS_MIGRATION.py

@@ -73,7 +73,7 @@ old_crawlers.r_serv_onion = r_crawler
 
 # CREATE FUNCTION BY DB/FEATURES
 
-# /!\ ISSUE WITH FILE DUPLICATES => NEED TO BE REFACTORED
+# /!\ TODO MIGRATE DUPLICATES
 
 
 def get_item_date(item_id):
@@ -670,28 +670,38 @@ def get_subtype_object(obj_type, subtype, obj_id):
         return Username(obj_id, subtype)
 
 def migrate_subtype_obj(Obj, obj_type, subtype, obj_id):
-    first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id)
+    # first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id)
-    last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id)
+    # last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id)
 
     # dates
     for item_id in get_item_correlation_obj(obj_type, subtype, obj_id):
         date = get_item_date(item_id)
         Obj.add(date, item_id)
 
+
 dict_obj_subtypes = {'cryptocurrency': ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'],
                      'pgpdump': ['key', 'mail', 'name'],
                      'username': ['telegram', 'twitter', 'jabber']}
 
 def subtypes_obj_migration():
-    print('SUBPTYPE MIGRATION...')
+    print('SUBTYPE MIGRATION...')
+    pgp_symmetrical_key = '0x0000000000000000'
 
     for obj_type in dict_obj_subtypes:
         print(f'{obj_type} MIGRATION...')
         for subtype in dict_obj_subtypes[obj_type]:
             for obj_id in get_all_subtype_id(obj_type, subtype):
+                if obj_type == 'pgp' and subtype == 'key' and obj_id == pgp_symmetrical_key:
+                    pass
+                else:
                     Obj = get_subtype_object(obj_type, subtype, obj_id)
                     migrate_subtype_obj(Obj, obj_type, subtype, obj_id)
 
+    # ADD PGP Symmetrical tag to item
+    for item_id in get_item_correlation_obj('pgpdump', 'key', pgp_symmetrical_key):
+        item = Items.Item(item_id)
+        item.add_tag(f'infoleak:automatic-detection="pgp-symmetric";{item_id}')  # TODO SELECT TAG
+
 # # # # # # # # # # # # # # # #
 #       STATISTICS
 #
@@ -702,10 +712,9 @@ def get_all_provider():
     return r_serv_trend.smembers('all_provider_set')
 
 def get_item_source_stats_by_date(date, source):
-    stats = {}
+    stats = {'num': r_serv_trend.hget(f'{source}_num', date),
-    stats['num'] = r_serv_trend.hget(f'{source}_num', date)
+             'size': r_serv_trend.hget(f'{source}_size', date),
-    stats['size'] = r_serv_trend.hget(f'{source}_size', date)
+             'avg': r_serv_trend.hget(f'{source}_avg', date)}
-    stats['avg'] = r_serv_trend.hget(f'{source}_avg', date)
     return stats
 
 def get_item_stats_size_avg_by_date(date):

bin/IPAddress.py

@@ -74,7 +74,6 @@ if __name__ == '__main__':
         print('Please provide a list of valid IP addresses')
         sys.exit(0)
 
-
     # Sent to the logging a description of the module
     publisher.info("Run IP module")
 

bin/export/AILObjects.py

@@ -4,7 +4,6 @@
 import os
 import sys
 import uuid
-import redis
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))

bin/export/MispImport.py

@@ -42,7 +42,7 @@ def get_global_id_from_id(global_id):
         obj_meta['id'] = global_id[2]
     else:
         obj_meta['type'] = global_id[0]
-        obj_meta['subtype'] = None
+        obj_meta['subtype'] = ''
         obj_meta['id'] = global_id[1]
     return obj_meta
 

bin/lib/Correlate_object.py

@@ -433,25 +433,7 @@ def get_obj_global_id(obj_type, obj_id, obj_sub_type=None):
 
         return '{}:{}'.format(obj_type, obj_id)
 
-def get_global_id_from_id(global_id):
-    obj_meta = {}
-    global_id = global_id.split(':', 3)
-    if len(global_id) > 2:
-        obj_meta['type'] = global_id[0]
-        obj_meta['subtype'] = global_id[1]
-        obj_meta['id'] = global_id[2]
-    else:
-        obj_meta['type'] = global_id[0]
-        obj_meta['subtype'] = None
-        obj_meta['id'] = global_id[1]
-    return obj_meta
 
-# used by UI
-def get_obj_str_type_subtype(obj_type, obj_subtype):
-    if obj_subtype:
-        return '{};{}'.format(obj_type, obj_subtype)
-    else:
-        return obj_type
 
 def sanitise_correlation_names(correlation_names):
     '''

bin/lib/Duplicate.py

@@ -36,7 +36,6 @@ config_loader = None
 # Duplicates domains != Duplicates items
 
 
-
 def get_ssdeep_hash(content):
     return ssdeep.hash(content)
 
@@ -100,8 +99,6 @@ def add_obj_duplicate(algo, hash, similarity, obj_type, subtype, id, date_ymonth
     _add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id)
 
 
-
-
 def get_last_x_month_dates(nb_months):
     now = datetime.datetime.now()
     result = [now.strftime("%Y%m")]
@@ -111,20 +108,7 @@ def get_last_x_month_dates(nb_months):
     return result
 
 
-
 if __name__ == '__main__':
     res = get_last_x_month_dates(7)
     print(res)
 
-
-
-
-
-
-
-
-
-
-
-
-#################################

bin/lib/Tag.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-import os
-import sys
 import redis
 import datetime
 
+import os
+import json
+import sys
+
+from glob import glob
+
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
@@ -24,31 +28,35 @@ config_loader = None
 
 #### CORE FUNCTIONS ####
 
+# # # # UNSAFE TAGS # # # #
+
 def build_unsafe_tags():
-    unsafe_tags = set()
+    tags = set()
-    ## CE content
+    # CE content
-    unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
+    tags.add('dark-web:topic="pornography-child-exploitation"')
     # add copine-scale tags
     taxonomies = Taxonomies()
     copine_scale = taxonomies.get('copine-scale')
     if copine_scale:
         for tag in copine_scale.machinetags():
-            unsafe_tags.add(tag)
+            tags.add(tag)
-    return unsafe_tags
+    return tags
+
+def is_tags_safe(ltags):
+    """
+    Check if a list of tags contain unsafe tags (CE, ...)
+    :param ltags: list of tags
+    :type ltags: list
+    :return: is a tag in the set unsafe
+    :rtype: boolean
+    """
+    return unsafe_tags.isdisjoint(ltags)
+
 
 # set of unsafe tags
 unsafe_tags = build_unsafe_tags()
 
-def is_tags_safe(ltags):
+# - - - UNSAFE TAGS - - - #
-    '''
-    Check if a list of tags contain an unsafe tag (CE, ...)
-
-    :param ltags: list of tags
-    :type ltags: list
-    :return: is a tag in the unsafe set
-    :rtype: boolean
-    '''
-    return unsafe_tags.isdisjoint(ltags)
 
 # # TODO: verify tags + object_type
 # get set_keys: intersection
@@ -65,7 +73,405 @@ def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None):
 def get_obj_by_tag(key_tag):
     return r_tags.smembers(key_tag)
 
-##-- CORE FUNCTIONS --##
+# -- CORE FUNCTIONS -- #
+
+
+#### Taxonomies ####
+
+TAXONOMIES = {}
+def load_taxonomies():
+    global TAXONOMIES
+    manifest = os.path.join(os.environ['AIL_HOME'], 'files/misp-taxonomies/MANIFEST.json')
+    TAXONOMIES = Taxonomies(manifest_path=manifest)
+
+
+load_taxonomies()
+
+def get_taxonomies():
+    return TAXONOMIES.keys()
+
+# r_tags.sadd(f'active_taxonomies', taxonomy)
+def is_taxonomy_enabled(taxonomy):
+    # enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
+    try:
+        enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
+    except redis.exceptions.ResponseError:
+        enabled = False
+    return enabled
+
+def enable_taxonomy(taxonomy):
+    r_tags.sadd('taxonomies:enabled', taxonomy)
+
+def disable_taxonomy(taxonomy):
+    r_tags.srem('taxonomies:enabled', taxonomy)
+
+def exists_taxonomy(taxonomy):
+    return TAXONOMIES.get(taxonomy) is not None
+
+def get_taxonomy_description(taxonomy):
+    return TAXONOMIES.get(taxonomy).description
+
+def get_taxonomy_name(taxonomy):
+    return TAXONOMIES.get(taxonomy).name
+
+def get_taxonomy_predicates(taxonomy):
+    meta = {}
+    predicates = taxonomy.predicates
+    for predicate in predicates:
+        meta[predicate] = {}
+        expanded = predicates[predicate].expanded
+        if expanded:
+            meta[predicate]['expanded'] = expanded
+        description = predicates[predicate].description
+        if description:
+            meta[predicate]['description'] = description
+    return meta
+
+def get_taxonomy_refs(taxonomy):
+    return TAXONOMIES.get(taxonomy).refs
+
+def get_taxonomy_version(taxonomy):
+    return TAXONOMIES.get(taxonomy).version
+
+def get_taxonomy_tags(taxonomy, enabled=False):
+    taxonomy_obj = TAXONOMIES.get(taxonomy)
+    tags = []
+    for p, content in taxonomy_obj.items():
+        if content:
+            for k, entry in content.items():
+                tag = f'{taxonomy_obj.name}:{p}="{k}"'
+                expanded = entry.expanded
+                meta_tag = {'tag': tag, 'expanded': expanded}
+                if enabled:
+                    meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag)
+                tags.append(meta_tag)
+        else:
+            tag = f'{taxonomy_obj.name}:{p}'
+            expanded = content.expanded
+            meta_tag = {'tag': tag, 'expanded': expanded}
+            if enabled:
+                meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag)
+            tags.append(meta_tag)
+    return tags
+
+# TODO GET ACTIVE TAGS
+# TODO GET IF TAXONOMY IS ACTIVATED
+def get_taxonomy_meta(taxonomy_name, enabled=False, enabled_tags=False, nb_active_tags=False, predicates=False, tags=False, expanded=True):
+    meta = {}
+    if not exists_taxonomy(taxonomy_name):
+        return meta
+    taxonomy = TAXONOMIES.get(taxonomy_name)
+    meta['description'] = taxonomy.description
+    meta['name'] = taxonomy.name
+    meta['version'] = taxonomy.version
+    if enabled:
+        meta['enabled'] = is_taxonomy_enabled(taxonomy_name)
+    if predicates:
+        meta['predicates'] = get_taxonomy_predicates(taxonomy)
+    if taxonomy.expanded:
+        meta['expanded'] = taxonomy.expanded
+    if taxonomy.refs:
+        meta['refs'] = taxonomy.refs
+
+    # TODO PERF SAVE IN DB ?????
+    if tags:
+        if expanded:
+            meta['tags'] = get_taxonomy_tags(taxonomy_name, enabled=enabled_tags)
+        else:
+            meta['tags'] = taxonomy.machinetags()
+        meta['nb_tags'] = len(meta['tags'])
+    if nb_active_tags:
+        meta['nb_active_tags'] = get_taxonomy_nb_tags_enabled(taxonomy_name)
+        if not tags:
+            meta['nb_tags'] = len(taxonomy.machinetags())
+    return meta
+
+def get_taxonomies_meta():
+    meta = {}
+    for taxonomy in get_taxonomies():
+        meta[taxonomy] = get_taxonomy_meta(taxonomy, enabled=True, nb_active_tags=True)
+    return meta
+
+    #### Enabled Tags ####
+# r_tags.sadd('active_taxonomies', taxonomy)
+def get_taxonomy_tags_enabled(taxonomy):
+    return r_tags.smembers(f'taxonomy:tags:enabled:{taxonomy}')
+
+def get_taxonomy_nb_tags_enabled(taxonomy):
+    return r_tags.scard(f'taxonomy:tags:enabled:{taxonomy}')
+
+def is_taxonomy_tag_enabled(taxonomy, tag):
+    # r_tags.sismember('list_tags', tag)
+    try:
+        enabled = r_tags.sismember(f'taxonomy:tags:enabled:{taxonomy}', tag)
+    except redis.exceptions.ResponseError:
+        enabled = False
+    return enabled
+
+def add_taxonomy_tag_enabled(taxonomy, tag):
+    r_tags.sadd(f'taxonomy:tags:enabled:{taxonomy}', tag)
+
+def remove_taxonomy_tag_enabled(taxonomy, tag):
+    r_tags.srem(f'taxonomy:tags:enabled:{taxonomy}', tag)
+
+def disable_taxonomy_tags_enabled(taxonomy):
+    r_tags.delete(f'taxonomy:tags:enabled:{taxonomy}')
+
+def update_taxonomy_tag_enabled(taxonomy, tags):
+    if not tags:
+        return None
+    enable_taxonomy(taxonomy)
+    tags = set(tags)
+    enabled_tags = get_taxonomy_tags_enabled(taxonomy)
+    if tags != enabled_tags:
+        # disable tags
+        for tag in enabled_tags.difference(tags):
+            remove_taxonomy_tag_enabled(taxonomy, tag)
+        # enable tags
+        for tag in tags:
+            add_taxonomy_tag_enabled(taxonomy, tag)
+
+def api_update_taxonomy_tag_enabled(data):
+    taxonomy = data.get('taxonomy')
+    if not exists_taxonomy(taxonomy):
+        return {'error': f'taxonomy {taxonomy} not found'}, 404
+    tags = data.get('tags', [])
+    taxonomy_tags = set(TAXONOMIES.get(taxonomy).machinetags())
+    for tag in tags:
+        if tag not in taxonomy_tags:
+            return {'error': f'tag {tag} not found'}, 404
+    update_taxonomy_tag_enabled(taxonomy, tags)
+
+def enable_taxonomy_tags(taxonomy):
+    enable_taxonomy(taxonomy)
+    for tag in TAXONOMIES.get(taxonomy).machinetags():
+        add_taxonomy_tag_enabled(taxonomy, tag)
+
+def api_enable_taxonomy_tags(data):
+    taxonomy = data.get('taxonomy')
+    if not exists_taxonomy(taxonomy):
+        return {'error': f'taxonomy {taxonomy} not found'}, 404
+    enable_taxonomy_tags(taxonomy)
+
+def disable_taxonomy_tags(taxonomy):
+    disable_taxonomy(taxonomy)
+    disable_taxonomy_tags_enabled(taxonomy)
+
+def api_disable_taxonomy_tags(data):
+    taxonomy = data.get('taxonomy')
+    if not exists_taxonomy(taxonomy):
+        return {'error': f'taxonomy {taxonomy} not found'}, 404
+    disable_taxonomy_tags(taxonomy)
+
+    # -- Enabled Tags -- #
+
+# -- Taxonomies -- #
+
+#### GALAXIES ####
+
+#
+# var galaxy = galaxy type
+#
+
+
+GALAXIES = {}
+CLUSTERS = {}
+def load_galaxies():
+    global GALAXIES
+    galaxies = []
+    root_dir_galaxies = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/galaxies')
+    for galaxy_file in glob(os.path.join(root_dir_galaxies, '*.json')):
+        with open(galaxy_file, 'r') as f:
+            galaxies.append(json.load(f))
+    GALAXIES = Galaxies(galaxies=galaxies)
+    global CLUSTERS
+    clusters = []
+    root_dir_clusters = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/clusters')
+    for cluster_file in glob(os.path.join(root_dir_clusters, '*.json')):
+        with open(cluster_file, 'r') as f:
+            clusters.append(json.load(f))
+    CLUSTERS = Clusters(clusters)
+
+
+# LOAD GALAXY + CLUSTERS
+load_galaxies()
+
+def get_galaxies():
+    return GALAXIES.keys()
+
+def get_galaxy(galaxy_name):
+    return GALAXIES.get(galaxy_name)
+
+def exists_galaxy(galaxy):
+    return CLUSTERS.get(galaxy) is not None
+
+# r_tags.sadd('active_galaxies', galaxy)
+def is_galaxy_enabled(galaxy):
+    try:
+        enabled = r_tags.sismember('galaxies:enabled', galaxy)
+    except redis.exceptions.ResponseError:
+        enabled = False
+    return enabled
+
+def enable_galaxy(galaxy):
+    r_tags.sadd('galaxies:enabled', galaxy)
+
+def disable_galaxy(galaxy):
+    r_tags.srem('galaxies:enabled', galaxy)
+
+def get_galaxy_meta(galaxy_name, nb_active_tags=False):
+    galaxy = get_galaxy(galaxy_name)
+    meta = {'name': galaxy.name, 'namespace': galaxy.namespace, 'description': galaxy.description,
+            'type': galaxy.type, 'version': galaxy.version, 'enabled': is_galaxy_enabled(galaxy.type)}
+    icon = galaxy.icon
+    if icon == 'android' or icon == 'optin-monster' or icon == 'internet-explorer' or icon == 'btc':
+        meta['icon'] = f'fab fa-{icon}'
+    else:
+        meta['icon'] = f'fas fa-{icon}'
+    if nb_active_tags:
+        meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy)
+        meta['nb_tags'] = len(get_galaxy_tags(galaxy.type))
+    return meta
+
+def get_galaxies_meta():
+    galaxies = []
+    for galaxy_name in get_galaxies():
+        galaxies.append(get_galaxy_meta(galaxy_name, nb_active_tags=True))
+    return galaxies
+
+def get_galaxy_tag_meta(galaxy_type, tag):
+    cluster = get_cluster(galaxy_type)
+    if not cluster:
+        return {}
+    try:
+        tag_val = tag.rsplit('=', 1)[1][1:-1]
+    except:
+        return {}
+    cluster_val = cluster.cluster_values.get(tag_val)
+    if not cluster_val:
+        return {}
+    meta = cluster_val.to_dict()
+    if 'meta' in meta:
+        meta['meta'] = json.loads(meta.get('meta').to_json())
+        meta['meta'] = json.dumps(meta['meta'], ensure_ascii=False, indent=4)
+    meta['tag'] = f'misp-galaxy:{galaxy_type}="{cluster_val.value}"'
+    meta['enabled'] = is_galaxy_tag_enabled(galaxy_type, meta['tag'])
+    return meta
+
+
+def get_clusters():
+    return CLUSTERS.keys()
+
+def get_cluster(cluster_type):
+    return CLUSTERS.get(cluster_type)
+
+def get_galaxy_tags(galaxy_type):
+    cluster = get_cluster(galaxy_type)
+    return cluster.machinetags()
+
+# TODO synonym
+def get_cluster_tags(cluster_type, enabled=False):
+    tags = []
+    cluster = get_cluster(cluster_type)
+    for cluster_val in cluster.values():
+        tag = f'misp-galaxy:{cluster_type}="{cluster_val.value}"'
+        meta_tag = {'tag': tag, 'description': cluster_val.description}
+        if enabled:
+            meta_tag['enabled'] = is_galaxy_tag_enabled(cluster_type, tag)
+        synonyms = cluster_val.meta.synonyms
+        if not synonyms:
+            synonyms = []
+        meta_tag['synonyms'] = synonyms
+        tags.append(meta_tag)
+    return tags
+
+def get_cluster_meta(cluster_type, tags=False, enabled=False):
+    cluster = get_cluster(cluster_type)
+    if not cluster:
+        return {}
+    meta = {'name': cluster.name, 'type': cluster.type, 'source': cluster.source,
+            'authors': cluster.authors, 'description': cluster.description, 'version': cluster.version,
+            'category': cluster.category}
+    if enabled:
+        meta['enabled'] = is_galaxy_enabled(cluster_type)
+    if tags:
+        meta['tags'] = get_cluster_tags(cluster_type, enabled=enabled)
+
+    return meta
+
+    #### Enabled Tags ####
+
+def get_galaxy_tags_enabled(galaxy):
+    return r_tags.smembers(f'galaxy:tags:enabled:{galaxy}')
+
+def get_galaxy_nb_tags_enabled(galaxy):
+    return r_tags.scard(f'galaxy:tags:enabled:{galaxy}')
+
+def is_galaxy_tag_enabled(galaxy, tag):
+    try:
+        enabled = r_tags.sismember(f'galaxy:tags:enabled:{galaxy}', tag)
+    except redis.exceptions.ResponseError:
+        enabled = False
+    return enabled
+
+def add_galaxy_tag_enabled(galaxy, tag):
+    r_tags.sadd(f'galaxy:tags:enabled:{galaxy}', tag)
+
+def remove_galaxy_tag_enabled(galaxy, tag):
+    r_tags.srem(f'galaxy:tags:enabled:{galaxy}', tag)
+
+def disable_galaxy_tags_enabled(galaxy):
+    r_tags.delete(f'galaxy:tags:enabled:{galaxy}')
+
+def update_galaxy_tag_enabled(galaxy, tags):
+    if not tags:
+        return None
+    enable_galaxy(galaxy)
+    tags = set(tags)
+    enabled_tags = get_galaxy_tags_enabled(galaxy)
+    if tags != enabled_tags:
+        # disable tags
+        for tag in enabled_tags.difference(tags):
+            remove_galaxy_tag_enabled(galaxy, tag)
+        # enable tags
+        for tag in tags:
+            add_galaxy_tag_enabled(galaxy, tag)
+
+def api_update_galaxy_tag_enabled(data):
+    galaxy = data.get('galaxy')
+    if not exists_galaxy(galaxy):
+        return {'error': f'galaxy {galaxy} not found'}, 404
+    tags = data.get('tags', [])
+    galaxy_tags = set(get_galaxy_tags(galaxy))
+    for tag in tags:
+        if tag not in galaxy_tags:
+            return {'error': f'tag {tag} not found'}, 404
+    update_galaxy_tag_enabled(galaxy, tags)
+
+def enable_galaxy_tags(galaxy):
+    enable_galaxy(galaxy)
+    for tag in get_galaxy_tags(galaxy):
+        add_galaxy_tag_enabled(galaxy, tag)
+
+def api_enable_galaxy_tags(data):
+    galaxy = data.get('galaxy')
+    if not exists_galaxy(galaxy):
+        return {'error': f'galaxy {galaxy} not found'}, 404
+    enable_galaxy_tags(galaxy)
+
+def disable_galaxy_tags(galaxy):
+    disable_galaxy(galaxy)
+    disable_galaxy_tags_enabled(galaxy)
+
+def api_disable_galaxy_tags(data):
+    galaxy = data.get('galaxy')
+    if not exists_galaxy(galaxy):
+        return {'error': f'galaxy {galaxy} not found'}, 404
+    disable_galaxy_tags(galaxy)
+
+    # -- Enabled Tags -- #
+
+# -- GALAXIES -- #
 
 ################################################################################
 ################################################################################
@@ -74,38 +480,44 @@ def get_obj_by_tag(key_tag):
 ################################################################################
 
 def is_obj_tagged(obj_type, obj_id, subtype=''):
-    '''
+    """
     Check if a object is tagged
 
-    :param object_id: object id
+    :param obj_type: object type
-    :type domain: str
+    :type obj_type: str
+    :param subtype: object subtype
+    :type subtype: str
+    :param obj_id: object ID
+    :type obj_id: str
 
     :return: is object tagged
     :rtype: boolean
-    '''
+    """
     return r_tags.exists(f'tag:{obj_type}:{subtype}:{obj_id}')
 
 def is_obj_tagged_by_tag(obj_type, obj_id, tag, subtype=''):
-    '''
+    """
-    Check if a object is tagged
+    Check if a object is tagged  by a specified tag
 
-    :param object_id: object id
+    :param obj_type: object type
-    :type domain: str
+    :type obj_type: str
-    :param tag: object type
+    :param subtype: object subtype
-    :type domain: str
+    :type subtype: str
+    :param obj_id: object ID
+    :type obj_id: str
+    :param tag: tag
+    :type tag: str
 
-    :return: is object tagged
+    :return: is object tagged by a specified tag
     :rtype: boolean
-    '''
+    """
     return r_tags.sismember(f'tag:{obj_type}:{subtype}:{obj_id}', tag)
 
 
-
-
 #
-#   f'tag:{obj_type}:{subtype}:{id}'              f'tag:{id}'
+#   'tag:{obj_type}:{subtype}:{id}'              'tag:{id}'
 #
-#   f'list_tags:{obj_type}:{subtype}'             f'list_tags:{obj_type}'
+#   'list_tags:{obj_type}:{subtype}'             'list_tags:{obj_type}'
 #
 #       graph tags by days ???????????????????????????????
 #
@@ -217,7 +629,7 @@ def get_tag_objects(obj_type, subtype='', date=''):
 def get_object_tags(obj_type, obj_id, subtype=''):
     return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}')
 
-def add_object_tag(tag, obj_type, id, subtype=''): #############################
+def add_object_tag(tag, obj_type, id, subtype=''):
     if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1:
         r_tags.sadd('list_tags', tag)
         r_tags.sadd(f'list_tags:{obj_type}', tag)
@@ -237,18 +649,28 @@ def add_object_tag(tag, obj_type, id, subtype=''): #############################
 
         r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1)
 
-def update_tag_global_by_obj_type(tag, object_type, subtype=''):
+# obj -> Object()
+def confirm_tag(tag, obj):
+    if tag.startswith('infoleak:automatic-detection'):
+        tag = tag.replace('automatic-detection', 'analyst-detection', 1)
+        obj.add_tag(tag)
+        return True
+    return False
+
+# FIXME
+# TODO REVIEW ME
+def update_tag_global_by_obj_type(tag, obj_type, subtype=''):
     tag_deleted = False
-    if object_type=='item':
+    if obj_type == 'item':
         if not r_tags.exists(f'tag_metadata:{tag}'):
             tag_deleted = True
     else:
-        if not r_tags.exists(f'{object_type}:{subtype}:{tag}'):
+        if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'):
             r_tags.srem(f'list_tags:{obj_type}:{subtype}', tag)
             # Iterate on all subtypes
             delete_global_obj_tag = True
             for obj_subtype in ail_core.get_object_all_subtypes():
-                if r_tags.exists(f'list_tags:{obj_type}:{subtype}'):
+                if r_tags.exists(f'list_tags:{obj_type}:{obj_subtype}'):
                     delete_global_obj_tag = False
                     break
             if delete_global_obj_tag:
@@ -280,6 +702,7 @@ def delete_object_tag(tag, obj_type, id, subtype=''):
 
 ################################################################################################################
 
+# TODO: REWRITE OLD
 # TODO: rewrite me
 # TODO: other objects
 def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1):
@@ -359,7 +782,7 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p
             for elem in l_tagged_obj:
                 if current_index > stop:
                     break
-                if start <= current_index and stop >= current_index:
+                if start <= current_index <= stop:
                     l_obj.append(elem)
                 current_index += 1
             l_tagged_obj = l_obj
@@ -375,102 +798,6 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p
         return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
 
 
-
-################################################################################
-################################################################################
-################################################################################
-################################################################################
-
-#### Taxonomies - Galaxies ####
-
-################################################################################
-# galaxies = Galaxies()
-# clusters = Clusters(skip_duplicates=True)
-#
-# list_all_tags = {}
-# for name, c in clusters.items(): #galaxy name + tags
-#     list_all_tags[name] = c
-#
-# list_galaxies = []
-# for g in galaxies.values():
-#     list_galaxies.append(g.to_json())
-#
-# list_clusters = []
-# for c in clusters.values():
-#     list_clusters.append(c.to_json())
-#
-# # tags numbers in galaxies
-# total_tags = {}
-# for name, tags in clusters.items(): #galaxie name + tags
-#     total_tags[name] = len(tags)
-################################################################################
-
-#### Taxonomies ####
-
-def get_taxonomy_tags_from_cluster(taxonomy_name):
-    taxonomies = Taxonomies()
-    taxonomy = taxonomies[taxonomy_name]
-    return taxonomy.machinetags()
-
-# TODO: ADD api handler
-def enable_taxonomy(taxonomy):
-    tags = get_taxonomy_tags_from_cluster(taxonomy)
-    r_tags.sadd('active_taxonomies', taxonomy)
-    for tag in tags:
-        r_tags.sadd(f'active_tag_{taxonomy}', tag)
-
-# def enable_taxonomy(taxonomie, enable_tags=True):
-#     '''
-#     Enable a taxonomy. (UI)
-#
-#     :param taxonomie: MISP taxonomy
-#     :type taxonomie: str
-#     :param enable_tags: crawled domain
-#     :type enable_tags: boolean
-#     '''
-#     taxonomies = Taxonomies()
-#     if enable_tags:
-#         taxonomie_info = taxonomies.get(taxonomie)
-#         if taxonomie_info:
-#             # activate taxonomie
-#             r_tags.sadd('active_taxonomies', taxonomie)
-#             # activate taxonomie tags
-#             for tag in taxonomie_info.machinetags():
-#                 r_tags.sadd('active_tag_{}'.format(taxonomie), tag)
-#                 #r_tags.sadd('active_taxonomies_tags', tag)
-#         else:
-#             print('Error: {}, please update pytaxonomies'.format(taxonomie))
-
-#### Galaxies ####
-
-def get_galaxy_tags_from_cluster(galaxy_name):
-    clusters = Clusters(skip_duplicates=True)
-    cluster = clusters[galaxy_name]
-    return cluster.machinetags()
-
-def get_galaxy_tags_with_sysnonym_from_cluster(galaxy_name):
-    tags = {}
-    clusters = Clusters(skip_duplicates=True)
-    cluster = clusters[galaxy_name]
-    for data in cluster.to_dict()['values']:
-        tag = f'misp-galaxy:{cluster.type}="{data.value}"'
-        synonyms = data.meta.synonyms
-        if not synonyms:
-            synonyms = []
-        tags[tag] = synonyms
-    return tags
-
-def enable_galaxy(galaxy):
-    tags = get_galaxy_tags_with_sysnonym_from_cluster(galaxy)
-    r_tags.sadd('active_galaxies', galaxy)
-    for tag in tags:
-        r_tags.sadd(f'active_tag_galaxies_{galaxy}', tag)
-        # synonyms
-        for synonym in tags[tag]:
-            r_tags.sadd(f'synonym_tag_{tag}', synonym)
-
-
-
 ################################################################################
 ################################################################################
 ################################################################################
@@ -490,9 +817,6 @@ def get_galaxy_from_tag(tag):
     except IndexError:
         return None
 
-def get_taxonomies():
-    return Taxonomies().keys()
-
 def is_taxonomie(taxonomie, taxonomies=[]):
     if not taxonomies:
         taxonomies = get_taxonomies()
@@ -579,11 +903,11 @@ def is_taxonomie_tag_enabled(taxonomie, tag):
     else:
         return False
 
-def is_galaxy_tag_enabled(galaxy, tag):
+# def is_galaxy_tag_enabled(galaxy, tag):
-    if tag in r_tags.smembers('active_tag_galaxies_' + galaxy):
+#     if tag in r_tags.smembers('active_tag_galaxies_' + galaxy):
-        return True
+#         return True
-    else:
+#     else:
-        return False
+#         return False
 
 def is_custom_tag_enabled(tag):
     return r_tags.sismember('tags:custom:enabled_tags', tag)
@@ -748,12 +1072,12 @@ def unpack_str_tags_list(str_tags_list):
         return []
 
 # used by modal
-def get_modal_add_tags(item_id, object_type='item'):
+def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
     '''
     Modal: add tags to domain or Paste
     '''
     return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
-            "object_id": item_id, "object_type": object_type}
+            "object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
 
 ######## NEW VERSION ########
 def create_custom_tag(tag):
@@ -790,6 +1114,27 @@ def get_all_tags():
 def get_all_obj_tags(obj_type):
     return list(r_tags.smembers(f'list_tags:{obj_type}'))
 
+# # # UI # # #
+
+def get_enabled_tags_with_synonyms_ui():
+    list_tags = []
+    for tag in get_all_tags():
+        t = tag.split(':')[0]
+        # add synonym
+        str_synonyms = ' - synonyms: '
+        if t == 'misp-galaxy':
+            synonyms = get_tag_synonyms(tag)
+            for synonym in synonyms:
+                str_synonyms = str_synonyms + synonym + ', '
+        # add real tag
+        if str_synonyms != ' - synonyms: ':
+            list_tags.append({'name': tag + str_synonyms, 'id': tag})
+        else:
+            list_tags.append({'name': tag, 'id': tag})
+    return list_tags
+
+# - - UI - - #
+
 ## Objects tags ##
 
 ###################################################################################
@@ -807,59 +1152,59 @@ def add_global_tag(tag, object_type=None):
     Create a set of all tags used in AIL (all + by object)
 
     :param tag: tag
-    :type domain: str
+    :type tag: str
     :param object_type: object type
-    :type domain: str
+    :type object_type: str
     '''
     r_tags.sadd('list_tags', tag)
     if object_type:
         r_tags.sadd('list_tags:{}'.format(object_type), tag)
 
-def add_obj_tags(object_id, object_type, tags=[], galaxy_tags=[]):
+def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
-    obj_date = get_obj_date(object_type, object_id)
     for tag in tags:
         if tag:
-            taxonomie = get_taxonomie_from_tag(tag)
+            taxonomy = get_taxonomie_from_tag(tag)
-            if is_taxonomie_tag_enabled(taxonomie, tag):
+            if is_taxonomie_tag_enabled(taxonomy, tag):
-                add_object_tag(tag, object_type, object_id)
+                add_object_tag(tag, object_type, object_id, object_subtype)
             else:
-                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
+                return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400
 
     for tag in galaxy_tags:
         if tag:
             galaxy = get_galaxy_from_tag(tag)
             if is_galaxy_tag_enabled(galaxy, tag):
-                add_object_tag(tag, object_type, object_id)
+                add_object_tag(tag, object_type, object_id, object_subtype)
             else:
-                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
+                return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400
 
 # TEMPLATE + API QUERY
-def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"):
+# WARNING CHECK IF OBJECT EXISTS
+def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item", object_subtype=''):
     res_dict = {}
-    if object_id == None:
+    if not object_id:
-        return ({'status': 'error', 'reason': 'object_id id not found'}, 404)
+        return {'status': 'error', 'reason': 'object_id id not found'}, 404
     if not tags and not galaxy_tags:
-        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+        return {'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400
-    if object_type not in ('item', 'domain', 'image', 'decoded'):  # # TODO: put me in another file
+    if object_type not in ail_core.get_all_objects():
-        return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
+        return {'status': 'error', 'reason': 'Incorrect object_type'}, 400
 
     # remove empty tags
     tags = list(filter(bool, tags))
     galaxy_tags = list(filter(bool, galaxy_tags))
 
-    res = add_obj_tags(object_id, object_type, tags=tags, galaxy_tags=galaxy_tags)
+    res = add_obj_tags(object_id, object_subtype, object_type, tags=tags, galaxy_tags=galaxy_tags)
     if res:
         return res
 
     res_dict['tags'] = tags + galaxy_tags
     res_dict['id'] = object_id
     res_dict['type'] = object_type
-    return (res_dict, 200)
+    return res_dict, 200
 
 # def add_tag(object_type, tag, object_id, obj_date=None):
 #     # new tag
 #     if not is_obj_tagged(object_id, tag):
-#         # # TODO: # FIXME: sanityze object_type
+#         # # TODO: # FIXME: sanitize object_type
 #         if obj_date:
 #             try:
 #                 obj_date = int(obj_date)
@@ -950,5 +1295,7 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type
     #r_serv_db.smembers('whitelist_hive')
 
 # if __name__ == '__main__':
-#     galaxy = 'infoleak'
+#     taxo = 'accessnow'
-#     get_taxonomy_tags_from_cluster(galaxy)
+#     # taxo = TAXONOMIES.get(taxo)
+#     res = is_taxonomy_tag_enabled(taxo, 'test')
+#     print(res)

bin/lib/ail_core.py

@@ -15,14 +15,15 @@ config_loader = ConfigLoader()
 
 config_loader = None
 
+AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}
+
 def get_ail_uuid():
     pass
 
 #### AIL OBJECTS ####
 
-# # TODO: check change paste => item
 def get_all_objects():
-    return ['cve', 'domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
+    return AIL_OBJECTS
 
 def get_object_all_subtypes(obj_type):
     if obj_type == 'cryptocurrency':

bin/lib/correlations_engine.py

@@ -69,10 +69,10 @@ def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type):
 def get_nb_correlations(obj_type, subtype, obj_id, filter_types=[]):
     if subtype is None:
         subtype = ''
-    nb_correlations = 0
+    obj_correlations = {}
     filter_types = sanityze_obj_correl_types(obj_type, filter_types)
     for correl_type in filter_types:
-        obj_correlations += get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type)
+        obj_correlations[correl_type] = get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type)
     return obj_correlations
 
 def get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type):

bin/lib/crawlers.py

@@ -546,7 +546,7 @@ def blacklist_domain(domain):
 
 def load_blacklist():
     try:
-        with open(os.path.join(os.environ['AIL_BIN'], 'torcrawler/blacklist.txt'), 'r') as f:
+        with open(os.path.join(os.environ['AIL_BIN'], 'crawlers/blacklist.txt'), 'r') as f:
             r_crawler.delete('blacklist:domain')
             lines = f.read().splitlines()
             for line in lines:

bin/lib/objects/CryptoCurrencies.py

@@ -92,7 +92,7 @@ class CryptoCurrency(AbstractSubtypeObject):
         meta = self._get_meta()
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags()
+        meta['tags'] = self.get_tags(r_list=True)
         return meta
 
 

bin/lib/objects/Cves.py

@@ -63,7 +63,7 @@ class Cve(AbstractDaterangeObject):
         meta = self._get_meta(options=options)
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags()
+        meta['tags'] = self.get_tags(r_list=True)
         return meta
 
     def add(self, date, item_id):

bin/lib/objects/Items.py

@@ -53,7 +53,7 @@ class Item(AbstractObject):
         super(Item, self).__init__('item', id)
 
     def exists(self):
-        return os.path.isfile(self.get_filename())
+        return item_basic.exist_item(self.id)
 
     def get_date(self, separator=False):
         """
@@ -123,6 +123,9 @@ class Item(AbstractObject):
         payload = {'raw': self.get_gzip_content(b64=True)}
         return payload
 
+    def get_parent(self):
+        return item_basic.get_item_parent(self.id)
+
     def set_father(self, father_id): # UPDATE KEYS ?????????????????????????????
         r_serv_metadata.sadd(f'paste_children:{father_id}', self.id)
         r_serv_metadata.hset(f'paste_metadata:{self.id}', 'father', father_id)
@@ -234,7 +237,7 @@ class Item(AbstractObject):
 
     def get_screenshot(self):
         s = self.get_correlation('screenshot')
-        if s:
+        if s.get('screenshot'):
             s = s['screenshot'].pop()[1:]
             return os.path.join(s[0:2], s[2:4], s[4:6], s[6:8], s[8:10], s[10:12], s[12:])
 
@@ -267,10 +270,13 @@ class Item(AbstractObject):
         if 'lines' in options:
             content = meta.get('content')
             meta['lines'] = self.get_meta_lines(content=content)
+        if 'parent' in options:
+            meta['parent'] = self.get_parent()
         if 'size' in options:
             meta['size'] = self.get_size(str=True)
-
+        if 'mimetype' in options:
-        # # TODO: ADD GET FATHER
+            content = meta.get('content')
+            meta['mimetype'] = self.get_mimetype(content=content)
 
         # meta['encoding'] = None
         return meta

bin/lib/objects/Pgps.py

@@ -45,7 +45,7 @@ class Pgp(AbstractSubtypeObject):
         meta = self._get_meta()
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags()
+        meta['tags'] = self.get_tags(r_list=True)
         return meta
 
     def get_link(self, flask_context=False):

bin/lib/objects/Usernames.py

@@ -67,7 +67,7 @@ class Username(AbstractSubtypeObject):
         meta = self._get_meta()
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags()
+        meta['tags'] = self.get_tags(r_list=True)
         return meta
 
     def get_misp_object(self):

bin/lib/objects/abstract_object.py

@@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib import Tag
 from lib import Duplicate
-from lib.correlations_engine import get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated
+from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
 from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
 
@@ -183,6 +183,9 @@ class AbstractObject(ABC):
         """
         return get_correlations(self.type, self.subtype, self.id)
 
+    def get_nb_correlations(self, filter_types=[]):
+        return get_nb_correlations(self.type, self.subtype, self.id, filter_types=filter_types)
+
     def add_correlation(self, type2, subtype2, id2):
         """
         Add object correlation

bin/lib/objects/abstract_subtype_object.py

@@ -49,6 +49,13 @@ class AbstractSubtypeObject(AbstractObject):
     def exists(self):
         return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}')
 
+    # def exists(self):
+    #     res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
+    #     if res is not None:
+    #         return True
+    #     else:
+    #         return False
+
     def get_first_seen(self, r_int=False):
         first_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen')
         if r_int:
@@ -70,7 +77,7 @@ class AbstractSubtypeObject(AbstractObject):
             return last_seen
 
     def get_nb_seen(self):
-        return r_metadata.scard(f'set_{self.type}_{self.subtype}:{self.id}')
+        return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id))
 
     # # TODO: CHECK RESULT
     def get_nb_seen_by_date(self, date_day):
@@ -81,22 +88,11 @@ class AbstractSubtypeObject(AbstractObject):
             return int(nb)
 
     def _get_meta(self):
-        meta_dict = {}
+        meta_dict = {'first_seen': self.get_first_seen(),
-        meta_dict['first_seen'] = self.get_first_seen()
+                     'last_seen': self.get_last_seen(),
-        meta_dict['last_seen'] = self.get_last_seen()
+                     'nb_seen': self.get_nb_seen()}
-        meta_dict['nb_seen'] = self.get_nb_seen()
         return meta_dict
 
-    # def exists(self):
-    #     res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
-    #     if res is not None:
-    #         return True
-    #     else:
-    #         return False
-
-    def exists(self):
-        return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}')
-
     def set_first_seen(self, first_seen):
         r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen', first_seen)
 
@@ -137,7 +133,7 @@ class AbstractSubtypeObject(AbstractObject):
         # daily
         r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
         # all subtypes
-        r_metadata.zincrby(f'{self.type}_all:{self.subtype}', self.id, 1)
+        r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
 
         #######################################################################
         #######################################################################

bin/lib/objects/ail_objects.py

@@ -109,7 +109,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
         meta["vt"]["status"] = obj.is_vt_enabled()
     # TAGS MODAL
     if obj.get_type() == 'screenshot' or obj.get_type() == 'decoded':
-        meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, object_type=obj.get_type())
+        meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
     return meta
 
 def get_ui_obj_tag_table_keys(obj_type):

bin/trackers/yara/ail-yara-rules

@@ -1 +1 @@
-Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4
+Subproject commit 3ccbec3145d3dec0ebf46fb996eb79d0932388d9

crawler_hidden_services_install.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-install_docker() {
-    # install docker
-    sudo apt install docker.io;
-
-    # pull splah docker
-    sudo docker pull scrapinghub/splash;
-}
-
-# install_python_requirement() {
-#     . ./AILENV/bin/activate;
-#     pip3 install -U -r crawler_requirements.txt;
-# }
-
-install_all() {
-    read -p "Do you want to install docker? (use local splash server) [y/n] " -n 1 -r
-    echo    # (optional) move to a new line
-    if [[ $REPLY =~ ^[Yy]$ ]]
-    then
-        install_docker;
-    fi
-}
-
-usage() {
-  echo "Usage: crawler_hidden_services_install.sh [-y | -n]" 1>&2;
-            echo "          -y: install docker"
-            echo "          -n: don't install docker"
-            echo ""
-            echo "example:"
-            echo "crawler_hidden_services_install.sh -y"
-            exit 1;
-}
-
-if [[ $1 == "" ]]; then
-    install_all;
-    exit;
-else
-  key="$1"
-  case $key in
-      "")
-      install_all;
-      ;;
-      -y|--yes)
-      install_docker;
-      install_python_requirement;
-      ;;
-      -n|--no)
-      install_python_requirement;
-      ;;
-      *)    # unknown option
-      usage;
-      ;;
-  esac
-fi

files/misp-galaxy

@@ -0,0 +1 @@
+Subproject commit aba1321b34e18122ec1825b54e2fc8176a4bd25c

files/misp-taxonomies

@@ -1 +1 @@
-Subproject commit f2fbd0e5fb02decb350232de16c21251db0e7517
+Subproject commit 7aeaa0b890e16f6d9b349f6db3577bf25a9a40ad

var/www/Flask_server.py

@@ -8,7 +8,6 @@ import json
 import time
 import uuid
 
-import redis
 import random
 import logging
 import logging.handlers
@@ -16,7 +15,6 @@ import logging.handlers
 from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
-import flask
 import importlib
 from os.path import join
 
@@ -132,6 +130,7 @@ def load_user(user_id):
 
 # ========= HEADER GENERATION ======== DEPRECATED
 
+
 # Get headers items that should be ignored (not displayed)
 toIgnoreModule = set()
 try:
@@ -166,7 +165,6 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
                 to_add_to_header_dico[module_name] = f.read()
 
 # create header.html
-complete_header = ""
 with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
     complete_header = f.read()
 modified_header = complete_header
@@ -250,21 +248,20 @@ def page_not_found(e):
 
 # ========== INITIAL taxonomies ============
 default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
-
 # enable default taxonomies
-for taxo in default_taxonomies:
+for taxonomy in default_taxonomies:
-    Tag.enable_taxonomy(taxo)
+    Tag.enable_taxonomy_tags(taxonomy)
 
 # ========== INITIAL tags auto export ============
-taxonomies = Taxonomies()
+# taxonomies = Taxonomies()
-
+#
-infoleak_tags = taxonomies.get('infoleak').machinetags()
+# infoleak_tags = taxonomies.get('infoleak').machinetags()
-infoleak_automatic_tags = []
+# infoleak_automatic_tags = []
-for tag in taxonomies.get('infoleak').machinetags():
+# for tag in taxonomies.get('infoleak').machinetags():
-    if tag.split('=')[0][:] == 'infoleak:automatic-detection':
+#     if tag.split('=')[0][:] == 'infoleak:automatic-detection':
-        r_serv_db.sadd('list_export_tags', tag)
+#         r_serv_db.sadd('list_export_tags', tag)
-
+#
-r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
+# r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
 # ============ MAIN ============
 
 if __name__ == "__main__":

var/www/blueprints/ail_2_ail_sync.py

@@ -8,7 +8,6 @@
 import os
 import sys
 import json
-import random
 
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
 from flask_login import login_required, current_user, login_user, logout_user
@@ -19,15 +18,15 @@ import Flask_config
 # Import Role_Manager
 from Role_Manager import login_admin, login_analyst, login_read_only
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+sys.path.append(os.environ['AIL_BIN'])
-import Tag
+##################################
+# Import Project packages
+##################################
+from core import ail_2_ail
+from lib import item_basic
+from lib import Tag
+from lib import Tracker
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
-import item_basic
-import Tracker
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core'))
-import ail_2_ail
 
 bootstrap_label = Flask_config.bootstrap_label
 
@@ -37,7 +36,6 @@ ail_2_ail_sync = Blueprint('ail_2_ail_sync', __name__, template_folder=os.path.j
 # ============ VARIABLES ============
 
 
-
 # ============ FUNCTIONS ============
 def api_validator(api_response):
     if api_response:

var/www/blueprints/correlation.py

@@ -136,7 +136,6 @@ def show_correlation():
                            "filter": filter_types, "filter_str": ",".join(filter_types),
                            "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, flask_context=True)
                            }
-            print(dict_object)
             if subtype:
                 dict_object["metadata"]['type_id'] = subtype
             dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
@@ -196,4 +195,4 @@ def subtype_search():
     obj_type = request.form.get('object_type')
     obj_subtype = request.form.get('object_subtype')
     obj_id = request.form.get('object_id')
-    return redirect(url_for('correlation.show_correlation', object_type=obj_type, type_id=obj_subtype, correlation_id=obj_id))
+    return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=obj_subtype, id=obj_id))

var/www/blueprints/hunters.py

@@ -19,9 +19,13 @@ import Flask_config
 # Import Role_Manager
 from Role_Manager import login_admin, login_analyst, login_read_only
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+sys.path.append(os.environ['AIL_BIN'])
-import item_basic
+##################################
-import Tracker
+# Import Project packages
+##################################
+from lib import item_basic
+from lib import Tracker
+
 
 bootstrap_label = Flask_config.bootstrap_label
 

var/www/blueprints/import_export.py

@@ -20,14 +20,16 @@ import Flask_config
 # Import Role_Manager
 from Role_Manager import login_admin, login_analyst, login_read_only
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export'))
+sys.path.append(os.environ['AIL_BIN'])
-import MispImport
+##################################
-import MispExport
+# Import Project packages
+##################################
+from export import MispExport
+from export import MispImport
+from export import AILObjects ####################### # # # # ########################## ## # ####################################
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+from lib.objects import ail_objects
-import Correlate_object
 
-import AILObjects
 
 # ============ BLUEPRINT ============
 import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
@@ -66,10 +68,11 @@ def import_object_file():
         map_uuid_global_id = MispImport.import_objs_from_file(filename)
         os.remove(filename)
         for obj_uuid in map_uuid_global_id:
-            dict_obj = Correlate_object.get_global_id_from_id(map_uuid_global_id[obj_uuid])
+            dict_obj = MispImport.get_global_id_from_id(map_uuid_global_id[obj_uuid])
+            obj = ail_objects.get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id'])
             dict_obj['uuid'] = obj_uuid
-            dict_obj['url'] = Correlate_object.get_item_url(dict_obj['type'], dict_obj['id'], correlation_type=dict_obj['subtype'])
+            dict_obj['url'] = obj.get_link(flask_context=True)
-            dict_obj['node'] = Correlate_object.get_correlation_node_icon(dict_obj['type'], correlation_type=dict_obj['subtype'], value=dict_obj['id'])
+            dict_obj['node'] = obj.get_svg_icon()
             all_imported_obj.append(dict_obj)
 
         if not all_imported_obj:
@@ -142,7 +145,9 @@ def export_object_file():
 
         for obj_dict in l_obj_invalid: # set uuid input
             obj_dict['uuid'] = str(uuid.uuid4())
-            obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
+            subtype = obj_dict.get('subtype', None)
+            if subtype:
+                obj_dict['type'] = f'{obj_dict["type"]};{subtype}'
 
         return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
                                l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)

var/www/blueprints/investigations_b.py

@@ -18,12 +18,13 @@ from Role_Manager import login_admin, login_analyst, login_read_only
 sys.path.append('modules')
 import Flask_config
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+sys.path.append(os.environ['AIL_BIN'])
-import Investigations
+##################################
+# Import Project packages
+##################################
+from lib import Investigations
 from lib.objects import ail_objects
-
+from lib import Tag
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-import Tag
 
 # ============ BLUEPRINT ============
 investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations'))

var/www/blueprints/objects_item.py

@@ -5,20 +5,21 @@
     Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
 '''
 
+import difflib
 import os
 import sys
-import json
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory
 from flask_login import login_required, current_user
 
 # Import Role_Manager
-from Role_Manager import login_admin, login_analyst, login_read_only
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
+from lib import ConfigLoader
 from lib import item_basic
 from lib.objects.Items import Item
 from lib import Tag
@@ -29,13 +30,25 @@ from export import Export
 objects_item = Blueprint('objects_item', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/item'))
 
 # ============ VARIABLES ============
+config_loader = ConfigLoader.ConfigLoader()
 bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
-
+DiffMaxLineLength = config_loader.get_config_int('Flask', 'DiffMaxLineLength')
+max_preview_modal = config_loader.get_config_int('Flask', 'max_preview_modal')
+SCREENSHOT_FOLDER = ConfigLoader.get_screenshots_dir()
+config_loader = None
 
 # ============ FUNCTIONS ============
 
 
 # ============= ROUTES ==============
+
+@objects_item.route('/screenshot/<path:filename>')
+@login_required
+@login_read_only
+@no_cache
+def screenshot(filename):
+    return send_from_directory(SCREENSHOT_FOLDER, f'{filename}.png', as_attachment=True)
+
 @objects_item.route("/object/item")
 @login_required
 @login_read_only
@@ -95,3 +108,60 @@ def item_download(): # # TODO: support post
         abort(404)
     item = Item(item_id)
     return send_file(item.get_raw_content(), download_name=item_id, as_attachment=True)
+
+@objects_item.route("/object/item/content/more")
+@login_required
+@login_read_only
+def item_content_more():
+    item_id = request.args.get('id', '')
+    item = Item(item_id)
+    item_content = item.get_content()
+    to_return = item_content[max_preview_modal-1:]
+    return to_return
+
+@objects_item.route("/object/item/diff")
+@login_required
+@login_analyst
+def object_item_diff():
+    id1 = request.args.get('s1', '')
+    id2 = request.args.get('s2', '')
+    item1 = Item(id1)
+    item2 = Item(id2)
+    item1_content = item1.get_content()
+    item2_content = item2.get_content()
+    i1_max_len = item1.get_meta_lines(content=item1_content)['max_length']
+    i2_max_len = item2.get_meta_lines(content=item2_content)['max_length']
+    if i1_max_len > DiffMaxLineLength or i2_max_len > DiffMaxLineLength:
+        return jsonify({'error': "Can't make the difference as the lines are too long."}), 400
+    lines1 = item1_content.splitlines()
+    lines2 = item2_content.splitlines()
+    htmldiff = difflib.HtmlDiff()
+    diff = htmldiff.make_file(lines1, lines2)
+    return diff
+
+@objects_item.route("/object/item/preview")
+@login_required
+@login_read_only
+def item_preview():
+    item_id = request.args.get('id')
+    if not item_id or not item_basic.exist_item(item_id):
+        abort(404)
+
+    item = Item(item_id)
+    meta = item.get_meta(options={'content', 'crawler', 'lines', 'mimetype', 'parent', 'size'})
+    initsize = len(meta['content'])
+    if len(meta['content']) > max_preview_modal:
+        meta['content'] = meta['content'][0:max_preview_modal]
+    meta['nb_correlations'] = item.get_nb_correlations()
+
+    misp_eventid = None  # TODO SHOW MISP EVENT
+    misp_url = None  # TODO SHOW MISP EVENT
+    hive_caseid = None  # TODO SHOW HIVE CASE
+    hive_url = None  # TODO SHOW HIVE CASE
+
+    return render_template("show_item_min.html", bootstrap_label=bootstrap_label,
+                           meta=meta, initsize=initsize,
+
+                           misp_eventid=misp_eventid, misp_url=misp_url,
+                           hive_caseid=hive_caseid, hive_url=hive_url)
+

var/www/blueprints/tags_ui.py

@@ -8,9 +8,8 @@
 import os
 import sys
 import json
-import random
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, abort
 from flask_login import login_required, current_user, login_user, logout_user
 
 sys.path.append('modules')
@@ -39,6 +38,140 @@ tags_ui = Blueprint('tags_ui', __name__, template_folder=os.path.join(os.environ
 
 
 # ============= ROUTES ==============
+@tags_ui.route('/tag/taxonomies')
+@login_required
+@login_read_only
+def tags_taxonomies():
+    taxonomies = Tag.get_taxonomies_meta()
+    return render_template("tags/taxonomies.html", taxonomies=taxonomies)
+
+@tags_ui.route('/tag/taxonomy')
+@login_required
+@login_read_only
+def tags_taxonomy():
+    taxonomy_name = request.args.get('taxonomy')
+    taxonomy = Tag.get_taxonomy_meta(taxonomy_name, enabled=True, predicates=True, tags=True, enabled_tags=True)
+    if not taxonomy:
+        abort(404)
+    return render_template("tags/taxonomy.html", taxonomy=taxonomy)
+
+@tags_ui.route('/tag/taxonomy/enable')
+@login_required
+@login_read_only
+def taxonomy_enable():
+    taxonomy = request.args.get('taxonomy')
+    res = Tag.api_enable_taxonomy_tags({'taxonomy': taxonomy})
+    if res:
+        return jsonify(res[0]), res[1]
+    else:
+        return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
+
+@tags_ui.route('/tag/taxonomy/disable')
+@login_required
+@login_read_only
+def taxonomy_disable():
+    taxonomy = request.args.get('taxonomy')
+    res = Tag.api_disable_taxonomy_tags({'taxonomy': taxonomy})
+    if res:
+        return jsonify(res[0]), res[1]
+    else:
+        return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
+
+@tags_ui.route('/tag/taxonomy/enable_tags')
+@login_required
+@login_read_only
+def taxonomy_enable_tags():
+    taxonomy = request.args.get('taxonomy')
+    tags = request.args.getlist('tags')
+    res = Tag.api_update_taxonomy_tag_enabled({'taxonomy': taxonomy, 'tags': tags})
+    if res:
+        return jsonify(res[0]), 1
+    else:
+        return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
+
+@tags_ui.route('/tag/galaxies')
+@login_required
+@login_read_only
+def tags_galaxies():
+    galaxies = Tag.get_galaxies_meta()
+    return render_template("tags/galaxies.html", galaxies=galaxies)
+
+@tags_ui.route('/tag/galaxy')
+@login_required
+@login_read_only
+def tags_galaxy():
+    galaxy_name = request.args.get('galaxy')
+    galaxy = Tag.get_cluster_meta(galaxy_name, enabled=True, tags=True)
+    if not galaxy:
+        abort(404)
+    return render_template("tags/galaxy.html", galaxy=galaxy)
+
+@tags_ui.route('/tag/galaxy/tag')
+@login_required
+@login_read_only
+def tags_galaxy_tag():
+    galaxy_type = request.args.get('galaxy')
+    tag = request.args.get('tag')
+    tag_meta = Tag.get_galaxy_tag_meta(galaxy_type, tag)
+    if not tag_meta:
+        abort(404)
+    return render_template("tags/galaxy_tag.html", galaxy=galaxy_type, tag=tag_meta)
+
+@tags_ui.route('/tag/galaxy/enable')
+@login_required
+@login_read_only
+def galaxy_enable():
+    galaxy = request.args.get('galaxy')
+    res = Tag.api_enable_galaxy_tags({'galaxy': galaxy})
+    if res:
+        return jsonify(res[0]), res[1]
+    else:
+        return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
+
+@tags_ui.route('/tag/galaxy/disable')
+@login_required
+@login_read_only
+def galaxy_disable():
+    galaxy = request.args.get('galaxy')
+    res = Tag.api_disable_galaxy_tags({'galaxy': galaxy})
+    if res:
+        return jsonify(res[0]), res[1]
+    else:
+        return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
+
+@tags_ui.route('/tag/galaxy/enable_tags')
+@login_required
+@login_read_only
+def galaxy_enable_tags():
+    galaxy = request.args.get('galaxy')
+    tags = request.args.getlist('tags')
+    res = Tag.api_update_galaxy_tag_enabled({'galaxy': galaxy, 'tags': tags})
+    if res:
+        return jsonify(res[0]), 1
+    else:
+        return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
+
+
+@tags_ui.route('/tag/enabled')
+@login_required
+@login_read_only
+def get_all_tags_enabled():
+    return jsonify(Tags.get_enabled_tags_with_synonyms_ui())
+
+@tags_ui.route('/tag/confirm')
+@login_required
+@login_read_only
+def tag_confirm():
+    tag = request.args.get('tag')
+    obj_type = request.args.get('type')
+    subtype = request.args.get('subtype', '')
+    obj_id = request.args.get('id', '')
+    obj = ail_objects.get_object(obj_type, subtype, obj_id)
+    if not obj.exists():
+        abort(404)
+    Tag.confirm_tag(tag, obj)
+    return redirect(obj.get_link(flask_context=True))
+
 @tags_ui.route('/tag/add_tags')
 @login_required
 @login_analyst
@@ -46,19 +179,20 @@ def add_tags():
 
     tags = request.args.get('tags')
     tagsgalaxies = request.args.get('tagsgalaxies')
-    object_id = request.args.get('object_id')
+    object_type = request.args.get('type')
-    object_type = request.args.get('object_type')
+    object_subtype = request.args.get('subtype')
-    subtype = '' # TODO: handle subtype object
+    object_id = request.args.get('id')
 
     list_tag = tags.split(',')
     list_tag_galaxies = tagsgalaxies.split(',')
 
-    res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, object_id=object_id, object_type=object_type)
+    res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies,
+                               object_id=object_id, object_type=object_type, object_subtype=object_subtype)
     # error
     if res[1] != 200:
         return str(res[0])
 
-    return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True))
+    return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True))
 
 @tags_ui.route('/tag/delete_tag')
 @login_required
@@ -95,7 +229,7 @@ def get_all_obj_tags():
     object_type = request.args.get('object_type')
     res = ail_objects.api_sanitize_object_type(object_type)
     if res:
-        return jsonify(res)
+        return jsonify(res[0]), res[1]
     return jsonify(Tag.get_all_obj_tags(object_type))
 
 @tags_ui.route('/tag/taxonomies/tags/enabled/json')
@@ -157,11 +291,11 @@ def tags_search_decoded():
     dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
     return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
 
-@tags_ui.route('/tag/search/image')
+@tags_ui.route('/tag/search/screenshot')
 @login_required
 @login_read_only
-def tags_search_images():
+def tags_search_screenshot():
-    object_type = 'image'
+    object_type = 'screenshot'
     dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
     return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
 
@@ -170,7 +304,7 @@ def tags_search_images():
 @login_read_only
 def get_obj_by_tags():
 
-    # # TODO: sanityze all
+    # # TODO: sanitize all
     object_type = request.args.get('object_type')
     subtype = ''  # TODO: handle subtype
     ltags = request.args.get('ltags')
@@ -198,19 +332,21 @@ def get_obj_by_tags():
     # page
     try:
         page = int(page)
-    except:
+    except (TypeError, ValueError):
         page = 1
 
+    # TODO REPLACE ME
     dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page)
+    print(dict_obj)
 
     if dict_obj['tagged_obj']:
         dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s",
                        "tagged_obj": [], "page": dict_obj['page'], "nb_pages": dict_obj['nb_pages'],
-                        "nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']}
+                       "nb_first_elem": dict_obj['nb_first_elem'], "nb_last_elem": dict_obj['nb_last_elem'],
+                       "nb_all_elem": dict_obj['nb_all_elem']}
 
         for obj_id in dict_obj['tagged_obj']:
             obj_metadata = ail_objects.get_object_meta(object_type, subtype, obj_id, flask_context=True)
-            #ail_objects.
             obj_metadata['id'] = obj_id
             dict_tagged["tagged_obj"].append(obj_metadata)
 

var/www/modules/Flask_config.py

@@ -29,6 +29,9 @@ r_serv = config_loader.get_redis_conn("Redis_Queues")
 r_cache = config_loader.get_redis_conn("Redis_Cache")
 r_serv_log = config_loader.get_redis_conn("Redis_Log")
 r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
+
+
+
 r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
 r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
 r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")

var/www/modules/Tags/Flask_Tags.py

@@ -13,8 +13,6 @@ from flask_login import login_required
 import json
 import datetime
 
-import Paste
-
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
 
@@ -25,38 +23,10 @@ from lib import Tag
 app = Flask_config.app
 baseUrl = Flask_config.baseUrl
 r_serv_tags = Flask_config.r_serv_tags
-r_serv_metadata = Flask_config.r_serv_metadata
-r_serv_statistics = Flask_config.r_serv_statistics
-max_preview_char = Flask_config.max_preview_char
-max_preview_modal = Flask_config.max_preview_modal
-bootstrap_label = Flask_config.bootstrap_label
-max_tags_result = Flask_config.max_tags_result
 
 Tags = Blueprint('Tags', __name__, template_folder='templates')
 
-galaxies = Galaxies()
-clusters = Clusters(skip_duplicates=True)
-
-list_all_tags = {}
-for name, c in clusters.items(): #galaxy name + tags
-    list_all_tags[name] = c
-
-list_galaxies = []
-for g in galaxies.values():
-    list_galaxies.append(g.to_json())
-
-list_clusters = []
-for c in clusters.values():
-    list_clusters.append(c.to_json())
-
-# tags numbers in galaxies
-total_tags = {}
-for name, tags in clusters.items(): #galaxie name + tags
-    total_tags[name] = len(tags)
-
 # ============ FUNCTIONS ============
-def one():
-    return 1
 
 # TODO:
 # TODO:
@@ -77,225 +47,17 @@ def get_tags_with_synonyms(tag):
     else:
         return {'name':tag,'id':tag}
 
-def get_item_date(item_filename):
-    l_directory = item_filename.split('/')
-    return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
 
-def substract_date(date_from, date_to):
-    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
-    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
-    delta = date_to - date_from # timedelta
-    l_date = []
-    for i in range(delta.days + 1):
-        date = date_from + datetime.timedelta(i)
-        l_date.append( date.strftime('%Y%m%d') )
-    return l_date
 
-def get_all_dates_range(date_from, date_to):
-    all_dates = {}
-    date_range = []
-    if date_from is not None and date_to is not None:
-        #change format
-        try:
-            if len(date_from) != 8:
-                date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
-                date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
-            date_range = substract_date(date_from, date_to)
-        except:
-            pass
 
-    if not date_range:
-        date_range.append(datetime.date.today().strftime("%Y%m%d"))
-        date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
-        date_to = date_from
-
-    else:
-        date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
-        date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
-    all_dates['date_from'] = date_from
-    all_dates['date_to'] = date_to
-    all_dates['date_range'] = date_range
-    return all_dates
-
-def get_last_seen_from_tags_list(list_tags):
-    min_last_seen = 99999999
-    for tag in list_tags:
-        tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-        if tag_last_seen:
-            tag_last_seen = int(tag_last_seen)
-            if tag_last_seen < min_last_seen:
-                min_last_seen = tag_last_seen
-    return str(min_last_seen)
 
 # ============= ROUTES ==============
 
-# @Tags.route("/tags/", methods=['GET'])
-# @login_required
-# @login_read_only
-# def Tags_page():
-#     date_from = request.args.get('date_from')
-#     date_to = request.args.get('date_to')
-#     tags = request.args.get('ltags')
-#
-#     if tags is None:
-#         dates = get_all_dates_range(date_from, date_to)
-#         return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
-#
-#     # unpack tags
-#     list_tags = tags.split(',')
-#     list_tag = []
-#     for tag in list_tags:
-#         list_tag.append(tag.replace('"','\"'))
-#
-#     #no search by date, use last_seen for  date_from/date_to
-#     if date_from is None and date_to is None and tags is not None:
-#         date_from = get_last_seen_from_tags_list(list_tags)
-#         date_to = date_from
-#
-#     # TODO verify input
-#
-#     dates = get_all_dates_range(date_from, date_to)
-#
-#     if(type(list_tags) is list):
-#         # no tag
-#         if list_tags is False:
-#             print('empty')
-#         # 1 tag
-#         elif len(list_tags) < 2:
-#             tagged_pastes = []
-#             for date in dates['date_range']:
-#                 tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
-#
-#         # 2 tags or more
-#         else:
-#             tagged_pastes = []
-#             for date in dates['date_range']:
-#                 tag_keys = []
-#                 for tag in list_tags:
-#                     tag_keys.append('{}:{}'.format(tag, date))
-#
-#                 if len(tag_keys) > 1:
-#                     daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
-#                 else:
-#                     daily_items = r_serv_tags.sinter(tag_keys[0])
-#                 tagged_pastes.extend(daily_items)
-#
-#     else :
-#         return 'INCORRECT INPUT'
-#
-#     all_content = []
-#     paste_date = []
-#     paste_linenum = []
-#     all_path = []
-#     allPastes = list(tagged_pastes)
-#     paste_tags = []
-#
-#     try:
-#         page = int(request.args.get('page'))
-#     except:
-#         page = 1
-#     if page <= 0:
-#         page = 1
-#     nb_page_max = len(tagged_pastes)/(max_tags_result)
-#     if not nb_page_max.is_integer():
-#         nb_page_max = int(nb_page_max)+1
-#     else:
-#         nb_page_max = int(nb_page_max)
-#     if page > nb_page_max:
-#         page = nb_page_max
-#     start = max_tags_result*(page -1)
-#     stop = max_tags_result*page
-#
-#     for path in allPastes[start:stop]:
-#         all_path.append(path)
-#         paste = Paste.Paste(path)
-#         content = paste.get_p_content()
-#         content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-#         all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
-#         curr_date = str(paste._get_p_date())
-#         curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-#         paste_date.append(curr_date)
-#         paste_linenum.append(paste.get_lines_info()[0])
-#         p_tags = r_serv_metadata.smembers('tag:'+path)
-#         complete_tags = []
-#         l_tags = []
-#         for tag in p_tags:
-#             complete_tag = tag
-#
-#             tag = tag.split('=')
-#             if len(tag) > 1:
-#                 if tag[1] != '':
-#                     tag = tag[1][1:-1]
-#                 # no value
-#                 else:
-#                     tag = tag[0][1:-1]
-#             # use for custom tags
-#             else:
-#                 tag = tag[0]
-#
-#             l_tags.append( (tag,complete_tag) )
-#
-#         paste_tags.append(l_tags)
-#
-#     if len(allPastes) > 10:
-#         finished = False
-#     else:
-#         finished = True
-#
-#     if len(list_tag) == 1:
-#         tag_nav=tags.replace('"', '').replace('=', '').replace(':', '')
-#     else:
-#         tag_nav='empty'
-#
-#     return render_template("Tags.html",
-#             all_path=all_path,
-#             tags=tags,
-#             tag_nav=tag_nav,
-#             list_tag = list_tag,
-#             date_from=dates['date_from'],
-#             date_to=dates['date_to'],
-#             page=page, nb_page_max=nb_page_max,
-#             paste_tags=paste_tags,
-#             bootstrap_label=bootstrap_label,
-#             content=all_content,
-#             paste_date=paste_date,
-#             paste_linenum=paste_linenum,
-#             char_to_display=max_preview_modal,
-#             finished=finished)
-
-
-@Tags.route("/Tags/get_all_tags")
-@login_required
-@login_read_only
-def get_all_tags():
-
-    all_tags = r_serv_tags.smembers('list_tags')
-
-    list_tags = []
-    for tag in all_tags:
-        t = tag.split(':')[0]
-        # add synonym
-        str_synonyms = ' - synonyms: '
-        if t == 'misp-galaxy':
-            synonyms = r_serv_tags.smembers('synonym_tag_' + tag)
-            for synonym in synonyms:
-                str_synonyms = str_synonyms + synonym + ', '
-        # add real tag
-        if str_synonyms != ' - synonyms: ':
-            list_tags.append({'name':tag + str_synonyms,'id':tag})
-        else:
-            list_tags.append({'name':tag,'id':tag})
-
-    return jsonify(list_tags)
-
 @Tags.route("/Tags/get_all_tags_taxonomies")
 @login_required
 @login_read_only
 def get_all_tags_taxonomies():
 
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
     active_taxonomie = r_serv_tags.smembers('active_taxonomies')
 
     list_tags = []
@@ -372,568 +134,36 @@ def get_tags_galaxy():
     else:
         return 'this galaxy is disable'
 
-@Tags.route("/Tags/remove_tag")
+# @Tags.route("/Tags/tag_validation")
-@login_required
+# @login_required
-@login_analyst
+# @login_analyst
-def remove_tag(): #TODO remove me , used by showpaste
+# def tag_validation():
+#
+#     path = request.args.get('paste')
+#     tag = request.args.get('tag')
+#     status = request.args.get('status')
+#
+#     if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag):
+#
+#         if status == 'tp':
+#             r_serv_statistics.sadd('tp:'+tag, path)
+#             r_serv_statistics.srem('fp:'+tag, path)
+#         else:
+#             r_serv_statistics.sadd('fp:'+tag, path)
+#             r_serv_statistics.srem('tp:'+tag, path)
+#
+#         return redirect(url_for('objects_item.showItem', id=path))
+#     else:
+#         return 'input error'
 
-    path = request.args.get('paste')
-    tag = request.args.get('tag')
 
-    res = Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
-    if res[1] != 200:
-        return str(res[0])
-    return redirect(url_for('objects_item.showItem', id=path))
 
-@Tags.route("/Tags/confirm_tag")
-@login_required
-@login_analyst
-def confirm_tag():
 
-    #TODO verify input
-    path = request.args.get('paste')
-    tag = request.args.get('tag')
 
-    if(tag[9:28] == 'automatic-detection'):
-        Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
 
-        tag = tag.replace('automatic-detection','analyst-detection', 1)
-        #add analyst tag
-        Tag.add_tag('item', tag, path)
 
-        return redirect(url_for('objects_item.showItem', id=path))
 
-    return 'incompatible tag'
 
-@Tags.route("/Tags/tag_validation")
-@login_required
-@login_analyst
-def tag_validation():
-
-    path = request.args.get('paste')
-    tag = request.args.get('tag')
-    status = request.args.get('status')
-
-    if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag):
-
-        if status == 'tp':
-            r_serv_statistics.sadd('tp:'+tag, path)
-            r_serv_statistics.srem('fp:'+tag, path)
-        else:
-            r_serv_statistics.sadd('fp:'+tag, path)
-            r_serv_statistics.srem('tp:'+tag, path)
-
-        return redirect(url_for('objects_item.showItem', id=path))
-    else:
-        return 'input error'
-
-@Tags.route("/Tags/taxonomies")
-@login_required
-@login_read_only
-def taxonomies():
-
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
-
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
-    id = []
-    name = []
-    description = []
-    version = []
-    enabled = []
-    n_tags = []
-
-    for taxonomie in list_taxonomies:
-        id.append(taxonomie)
-        name.append(taxonomies.get(taxonomie).name)
-        description.append(taxonomies.get(taxonomie).description)
-        version.append(taxonomies.get(taxonomie).version)
-        if taxonomie in active_taxonomies:
-            enabled.append(True)
-        else:
-            enabled.append(False)
-
-        n = str(r_serv_tags.scard('active_tag_' + taxonomie))
-        n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) )
-
-    return render_template("taxonomies.html",
-                            id=id,
-                            all_name = name,
-                            description = description,
-                            version = version,
-                            enabled = enabled,
-                            n_tags=n_tags)
-
-@Tags.route("/Tags/edit_taxonomie")
-@login_required
-@login_analyst
-def edit_taxonomie():
-
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
-    id = request.args.get('taxonomie')
-
-    #verify input
-    if id in list(taxonomies.keys()):
-        active_tag = r_serv_tags.smembers('active_tag_' + id)
-        list_tag = taxonomies.get(id).machinetags()
-        list_tag_desc = taxonomies.get(id).machinetags_expanded()
-
-        active_taxonomies = r_serv_tags.smembers('active_taxonomies')
-        if id in active_taxonomies:
-            active = True
-        else:
-            active = False
-
-        n = str(r_serv_tags.scard('active_tag_' + id))
-        badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
-
-        name = taxonomies.get(id).name
-        description = taxonomies.get(id).description
-        version = taxonomies.get(id).version
-
-        status = []
-        for tag in list_tag:
-            if tag in active_tag:
-                status.append(True)
-            else:
-                status.append(False)
-
-        return render_template("edit_taxonomie.html",
-            id=id,
-            name=name,
-            badge = badge,
-            description = description,
-            version = version,
-            active=active,
-            all_tags = list_tag,
-            list_tag_desc=list_tag_desc,
-            status = status)
-
-    else:
-        return 'INVALID TAXONOMIE'
-
-@Tags.route("/Tags/disable_taxonomie")
-@login_required
-@login_analyst
-def disable_taxonomie():
-
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
-    id = request.args.get('taxonomie')
-
-    if id in list_taxonomies:
-        r_serv_tags.srem('active_taxonomies', id)
-        for tag in taxonomies.get(id).machinetags():
-            r_serv_tags.srem('active_tag_' + id, tag)
-
-        return redirect(url_for('Tags.taxonomies'))
-
-    else:
-        return "INCORRECT INPUT"
-
-
-
-@Tags.route("/Tags/active_taxonomie")
-@login_required
-@login_analyst
-def active_taxonomie():
-
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
-    id = request.args.get('taxonomie')
-
-    # verify input
-    if id in list_taxonomies:
-        r_serv_tags.sadd('active_taxonomies', id)
-        for tag in taxonomies.get(id).machinetags():
-            r_serv_tags.sadd('active_tag_' + id, tag)
-
-        return redirect(url_for('Tags.taxonomies'))
-
-    else:
-        return "INCORRECT INPUT"
-
-@Tags.route("/Tags/edit_taxonomie_tag")
-@login_required
-@login_analyst
-def edit_taxonomie_tag():
-
-    taxonomies = Taxonomies()
-    list_taxonomies = list(taxonomies.keys())
-
-    arg1 = request.args.getlist('tag_enabled')
-    arg2 = request.args.getlist('tag_disabled')
-
-    id = request.args.get('taxonomie')
-
-    #verify input
-    if id in list_taxonomies:
-        list_tag = taxonomies.get(id).machinetags()
-
-        #check tags validity
-        if ( all(elem in list_tag  for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag  for elem in arg2) or (len(arg2) == 0) ):
-
-            active_tag = r_serv_tags.smembers('active_tag_' + id)
-
-            diff = list(set(arg1) ^ set(list_tag))
-
-            #remove tags
-            for tag in diff:
-                r_serv_tags.srem('active_tag_' + id, tag)
-
-            #all tags unchecked
-            if len(arg1) == 0 and len(arg2) == 0:
-                r_serv_tags.srem('active_taxonomies', id)
-
-            #add new tags
-            for tag in arg2:
-                r_serv_tags.sadd('active_taxonomies', id)
-                r_serv_tags.sadd('active_tag_' + id, tag)
-
-            return redirect(url_for('Tags.taxonomies'))
-        else:
-            return "INCORRECT INPUT"
-
-    else:
-        return "INCORRECT INPUT"
-
-@Tags.route("/Tags/galaxies")
-@login_required
-@login_read_only
-def galaxies():
-
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
-
-    name = []
-    icon = []
-    version = []
-    all_type = []
-    namespace = []
-    description = []
-    enabled = []
-    n_tags = []
-
-    for galaxie_json in list_galaxies:
-
-        galaxie = json.loads(galaxie_json)
-
-        name.append(galaxie['name'])
-        icon.append(galaxie['icon'])
-        version.append(galaxie['version'])
-        type = galaxie['type']
-        all_type.append(type)
-        namespace.append(galaxie['namespace'])
-        description.append(galaxie['description'])
-
-
-        if type in active_galaxies:
-            enabled.append(True)
-        else:
-            enabled.append(False)
-
-        n = str(r_serv_tags.scard('active_tag_galaxies_' + type))
-        n_tags.append(n + '/' + str(total_tags[type]) )
-
-    return render_template("galaxies.html",
-                            name=name,
-                            icon = icon,
-                            version = version,
-                            description = description,
-                            namespace = namespace,
-                            all_type = all_type,
-                            enabled = enabled,
-                            n_tags=n_tags)
-
-
-@Tags.route("/Tags/edit_galaxy")
-@login_required
-@login_analyst
-def edit_galaxy():
-
-    id = request.args.get('galaxy')
-
-    for clusters_json in list_clusters:
-
-        #get clusters
-        cluster = json.loads(clusters_json)
-
-        if cluster['type'] == id:
-
-            type = id
-            active_tag = r_serv_tags.smembers('active_tag_galaxies_' + type)
-
-            n = str(r_serv_tags.scard('active_tag_galaxies_' + type))
-            badge = n + '/' + str(total_tags[type])
-
-            name = cluster['name']
-            description = cluster['description']
-            version = cluster['version']
-            source = cluster['source']
-
-            val = cluster['values']
-
-            tags = []
-            for data in val:
-                try:
-                    meta = data['meta']
-                except KeyError:
-                    meta = []
-                tag_name = data['value']
-                tag_name = 'misp-galaxy:{}="{}"'.format(type, tag_name)
-                try:
-                    tag_description = data['description']
-                except KeyError:
-                    tag_description = ''
-
-                tags.append( (tag_name, tag_description, meta) )
-
-            status = []
-            for tag in tags:
-                if tag[0] in active_tag:
-                    status.append(True)
-                else:
-                    status.append(False)
-
-            active_galaxies = r_serv_tags.smembers('active_galaxies')
-            if id in active_galaxies:
-                active = True
-            else:
-                active = False
-
-            return render_template("edit_galaxy.html",
-                id = type,
-                name = name,
-                badge = badge,
-                description = description,
-                version = version,
-                active = active,
-                tags = tags,
-                status = status)
-
-
-    return 'INVALID GALAXY'
-
-
-@Tags.route("/Tags/active_galaxy")
-@login_required
-@login_analyst
-def active_galaxy():
-
-    id = request.args.get('galaxy')
-
-    # verify input
-    try:
-        l_tags = list_all_tags[id]
-    except KeyError:
-        return "INCORRECT INPUT"
-
-    r_serv_tags.sadd('active_galaxies', id)
-    for tag in l_tags:
-        r_serv_tags.sadd('active_tag_galaxies_' + id, 'misp-galaxy:{}="{}"'.format(id, tag))
-
-    #save synonyms
-    for clusters_json in list_clusters:
-
-        #get clusters
-        cluster = json.loads(clusters_json)
-
-        if cluster['type'] == id:
-
-            val = cluster['values']
-
-            tags = []
-            for data in val:
-                try:
-                    meta = data['meta']
-                    synonyms = meta['synonyms']
-                    tag_name = data['value']
-                    tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name)
-                    #save synonyms
-                    for synonym in synonyms:
-                        r_serv_tags.sadd('synonym_tag_' + tag_name, synonym)
-
-                except KeyError:
-                    pass
-
-            break
-
-    return redirect(url_for('Tags.galaxies'))
-
-
-@Tags.route("/Tags/disable_galaxy")
-@login_required
-@login_analyst
-def disable_galaxy():
-
-    id = request.args.get('galaxy')
-
-    # verify input
-    try:
-        l_tags = list_all_tags[id]
-    except KeyError:
-        return "INCORRECT INPUT"
-
-    r_serv_tags.srem('active_galaxies', id)
-    for tag in l_tags:
-        tag_name = 'misp-galaxy:{}="{}"'.format(id, tag)
-        r_serv_tags.srem('active_tag_galaxies_' + id, tag_name)
-        r_serv_tags.delete('synonym_tag_' + tag_name)
-
-    return redirect(url_for('Tags.galaxies'))
-
-
-@Tags.route("/Tags/edit_galaxy_tag")
-@login_required
-@login_analyst
-def edit_galaxy_tag():
-
-    arg1 = request.args.getlist('tag_enabled')
-    arg2 = request.args.getlist('tag_disabled')
-
-    id = request.args.get('galaxy')
-
-    #verify input
-    try:
-        l_tags = list_all_tags[id]
-    except KeyError:
-        return "INCORRECT INPUT"
-
-    #get full tags
-    list_tag = []
-    for tag in l_tags:
-        list_tag.append('misp-galaxy:{}="{}"'.format(id, tag))
-
-
-    #check tags validity
-    if ( all(elem in list_tag  for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag  for elem in arg2) or (len(arg2) == 0) ):
-
-        active_tag = r_serv_tags.smembers('active_tag_galaxies_' + id)
-
-        diff = list(set(arg1) ^ set(list_tag))
-
-        #remove tags
-        for tag in diff:
-            r_serv_tags.srem('active_tag_galaxies_' + id, tag)
-            r_serv_tags.delete('synonym_tag_' + tag)
-
-        #all tags unchecked
-        if len(arg1) == 0 and len(arg2) == 0:
-            r_serv_tags.srem('active_galaxies', id)
-
-        #add new tags
-        for tag in arg2:
-            r_serv_tags.sadd('active_galaxies', id)
-            r_serv_tags.sadd('active_tag_galaxies_' + id, tag)
-
-        #get tags synonyms
-        for clusters_json in list_clusters:
-
-            #get clusters
-            cluster = json.loads(clusters_json)
-
-            if cluster['type'] == id:
-
-                val = cluster['values']
-
-                tags = []
-                for data in val:
-                    try:
-                        meta = data['meta']
-                        synonyms = meta['synonyms']
-                        tag_name = data['value']
-                        tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name)
-                        if tag_name in arg2:
-                            #save synonyms
-                            for synonym in synonyms:
-                                r_serv_tags.sadd('synonym_tag_' + tag_name, synonym)
-
-                    except KeyError:
-                        pass
-                break
-
-        return redirect(url_for('Tags.galaxies'))
-
-    else:
-        return "INCORRECT INPUT"
-
-@Tags.route("/Tags/tag_galaxy_info")
-@login_required
-@login_read_only
-def tag_galaxy_info():
-
-    galaxy = request.args.get('galaxy')
-    tag = request.args.get('tag')
-
-    full_tag = tag
-    title = tag.split(':')[1]
-    tag = tag.split('=')[1]
-    tag = tag[1:-1]
-
-    #get clusters
-    for clusters_json in list_clusters:
-        cluster = json.loads(clusters_json)
-
-        if cluster['type'] == galaxy:
-            val = cluster['values']
-            source = cluster['source']
-
-            for data in val:
-                if tag == data['value']:
-                    try:
-                        description = data['description']
-                    except KeyError:
-                        description = ''
-                    if r_serv_tags.sismember('active_tag_galaxies_' + galaxy, full_tag):
-                        active = True
-                    else:
-                        active = False
-
-                    synonyms = []
-                    metadata = []
-                    list_metadata = []
-                    try:
-                        meta = data['meta']
-                        for key in meta:
-                            if key != 'synonyms':
-                                if type(meta[key]) is list:
-                                    for item in meta[key]:
-                                        list_metadata.append(key + ' :    ' + item)
-                                else:
-                                    list_metadata.append(key + ' :    ' + meta[key])
-                        try:
-                            synonyms = meta['synonyms']
-                            bool_synonyms = True
-                        except KeyError:
-                            synonyms = []
-                            bool_synonyms = False
-                    except KeyError:
-                        pass
-
-                    if synonyms:
-                        bool_synonyms = True
-                    else:
-                        bool_synonyms = False
-                    if list_metadata:
-                        metadata = True
-                    else:
-                        metadata = False
-
-                    return render_template("tag_galaxy_info.html",
-                                title = title,
-                                description = description,
-                                source = source,
-                                active = active,
-                                synonyms = synonyms,
-                                bool_synonyms = bool_synonyms,
-                                metadata = metadata,
-                                list_metadata = list_metadata)
-
-    return 'INVALID INPUT'
 
 # ========= REGISTRATION =========
 app.register_blueprint(Tags, url_prefix=baseUrl)

var/www/modules/Tags/templates/Tags.html

@@ -1,407 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <title>Tags - AIL</title>
-	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-  <!-- Core CSS -->
-	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
-
-  <!-- JS -->
-	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
-	<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
-	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
-	<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
-
-	<style>
-	.rotate{
-		-moz-transition: all 0.1s linear;
-		-webkit-transition: all 0.1s linear;
-		transition: all 0.1s linear;
-	}
-
-	.rotate.down{
-		-moz-transform:rotate(180deg);
-		-webkit-transform:rotate(180deg);
-		transform:rotate(180deg);
-	}
-	</style>
-
-</head>
-<body>
-
-  {% include 'nav_bar.html' %}
-
-	<!-- Modal -->
-  <div id="mymodal" class="modal fade" role="dialog">
-   <div class="modal-dialog modal-lg">
-
-     <!-- Modal content-->
-     <div id="mymodalcontent" class="modal-content">
-       <div id="mymodalbody" class="modal-body" max-width="850px">
-         <p>Loading paste information...</p>
-         <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
-       </div>
-       <div class="modal-footer">
-         <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
-         <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-       </div>
-     </div>
-   </div>
-  </div>
-
-	<div class="container-fluid">
-		<div class="row">
-
-			{% include 'tags/menu_sidebar.html' %}
-
-			<div class="col-12 col-lg-10" id="core_content">
-
-				<div class="card mb-3 mt-1">
-					<div class="card-header text-white bg-dark">
-						<h5 class="card-title">Search Tags by date range :</h5>
-					</div>
-					<div class="card-body">
-
-						<div class="row mb-3">
-							<div class="col-md-6">
-								<div class="input-group" id="date-range-from">
-									<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-									<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
-								</div>
-							</div>
-							<div class="col-md-6">
-								<div class="input-group" id="date-range-to">
-									<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-									<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
-								</div>
-							</div>
-						</div>
-
-						<div class="input-group mb-3">
-						  <div class="input-group-prepend">
-								<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
-									<i class="fas fa-eraser"></i>
-								</button>
-						  </div>
-							<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
-						</div>
-
-						<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
-							<i class="fas fa-search"></i> Search Tags
-						</button>
-
-					</div>
-				</div>
-
-				<div>
-					{%if all_path%}
-						<table class="table table-bordered table-hover" id="myTable_">
-				      <thead class="thead-dark">
-				        <tr>
-									<th>Date</th>
-				          <th style="max-width: 800px;">Path</th>
-				          <th># of lines</th>
-				          <th>Action</th>
-				        </tr>
-				      </thead>
-				      <tbody>
-
-				        {% for path in all_path %}
-				          <tr>
-										<td class="pb-0">{{ paste_date[loop.index0] }}</td>
-				            <td class="pb-0"><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{path}}" class="text-secondary">
-													<div style="line-height:0.9;">{{ path }}</div>
-												</a>
-				              <div class="mb-2">
-				                {% for tag in paste_tags[loop.index0] %}
-				                <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
-				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
-				                </a>
-				                {%  endfor %}
-				              </div>
-				            </td class="pb-0">
-				            <td class="pb-0">{{ paste_linenum[loop.index0] }}</td>
-				            <td class="pb-0"><p>
-											<span class="fas fa-info-circle" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span>
-											<button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsaveditem_min') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}">
-												<span class="fas fa-search-plus"></span>
-											</button></p>
-										</td>
-				          </tr>
-				        {%  endfor %}
-
-				      </tbody>
-				    </table>
-
-						<div class="d-flex justify-content-center border-top mt-2">
-							<nav class="mt-4" aria-label="...">
-								<ul class="pagination">
-									<li class="page-item {%if page==1%}disabled{%endif%}">
-										<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">Previous</a>
-									</li>
-
-									{%if page>3%}
-										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page=1&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">1</a></li>
-										<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
-										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>
-										<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
-									{%else%}
-									{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-2}}</a></li>{%endif%}
-									{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>{%endif%}
-									<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
-									{%endif%}
-
-									{%if nb_page_max-page>3%}
-										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page+1}}</a></li>
-										<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
-										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>
-									{%else%}
-										{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-2}}</a></li>{%endif%}
-										{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-1}}</a></li>{%endif%}
-										{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>{%endif%}
-									{%endif%}
-
-									<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
-										<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}" aria-disabled="true">Next</a>
-									</li>
-								</ul>
-							</nav>
-						</div>
-
-					{%endif%}
-				</div>
-
-				<div>
-					<br></br>
-					<a class="btn btn-light text-secondary" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
-							<i class="fas fa-wrench fa-2x"></i>
-							<br></br>
-							<span class="label-icon">Edit Taxonomies List </span>
-					</a>
-					<a class="btn btn-light text-secondary" href="{{ url_for('Tags.galaxies') }}" target="_blank">
-							<i class="fas fa-rocket fa-2x"></i>
-							<br></br>
-							<span class="label-icon">Edit Galaxies List</span>
-					</a>
-				</div>
-
-				<div>
-					<br></br>
-					<a class="btn btn-light text-secondary" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
-							<i class="fas fa-cogs fa-2x"></i>
-							<br></br>
-							<span class="label-icon">MISP and Hive, auto push</span>
-					</a>
-				</div>
-
-    	</div>
-		</div>
-	</div>
-
-</body>
-
-<script>
-var ltags;
-var search_table;
-var last_clicked_paste;
-var can_change_modal_content = true;
-
-$(document).ready(function(){
-	$("#nav_tags_search").removeClass("text-muted");
-	$("#nav_tags_search_item").addClass("active");
-	search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
-
-	// Use to bind the button with the new displayed data
-  // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
-  search_table.on( 'draw.dt', function () {
-    // Bind tooltip each time we draw a new page
-    $('[data-toggle="tooltip"]').tooltip();
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
-        get_html_and_update_modal(event, $(this));
-    });
-  } );
-
-	search_table.draw()
-
-
-	var valueData = [
-    {% for tag in list_tag %}
-      '{{tag|safe}}',
-    {%  endfor %}
-  ];
-
-	$.getJSON("{{ url_for('Tags.get_all_tags') }}",
-  function(data) {
-    ltags = $('#ltags').tagSuggest({
-      data: data,
-			value: valueData,
-      sortOrder: 'name',
-      maxDropHeight: 200,
-      name: 'ltags'
-    });
-  });
-
-	$('#date-range-from').dateRangePicker({
-			separator : ' to ',
-			getValue: function(){
-					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
-							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
-					else
-							return '';
-					},
-					setValue: function(s,s1,s2){
-							$('#date-range-from-input').val(s1);
-							$('#date-range-to-input').val(s2);
-					}
-	});
-	$('#date-range-to').dateRangePicker({
-			separator : ' to ',
-			getValue: function(){
-					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
-							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
-					else
-							return '';
-			},
-			setValue: function(s,s1,s2){
-					$('#date-range-from-input').val(s1);
-					$('#date-range-to-input').val(s2);
-			}
-	});
-});
-</script>
-
-<script>
-function searchTags() {
-	var data = ltags.getValue();
-	var date_from = $('#date-range-from-input').val();
-	var date_to =$('#date-range-to-input').val();
-	window.location.replace("{{ url_for('Tags.Tags_page') }}?date_from="+date_from+"&date_to="+date_to+"&ltags=" + data);
-}
-function emptyTags() {
-	ltags.clear();
-}
-
-function toggle_sidebar(){
-	if($('#nav_menu').is(':visible')){
-		$('#nav_menu').hide();
-		$('#side_menu').removeClass('border-right')
-		$('#side_menu').removeClass('col-lg-2')
-		$('#core_content').removeClass('col-lg-10')
-	}else{
-		$('#nav_menu').show();
-		$('#side_menu').addClass('border-right')
-		$('#side_menu').addClass('col-lg-2')
-		$('#core_content').addClass('col-lg-10')
-	}
-}
-</script>
-
-<!-- Dynamically update the modal -->
-<script>
-  // static data
-  var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
-  var complete_paste = null;
-  var char_to_display = {{ char_to_display }};
-  var start_index = 0;
-
-  // When the modal goes out, refresh it to normal content
-  $("#mymodal").on('hidden.bs.modal', function () {
-      can_change_modal_content = true;
-      $("#mymodalbody").html("<p>Loading paste information...</p>");
-      var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
-      $("#mymodalbody").append(loading_gif); // Show the loading GIF
-      $("#button_show_path").attr('href', '');
-      $("#button_show_path").hide();
-      complete_paste = null;
-      start_index = 0;
-  });
-
-  // Update the paste preview in the modal
-  function update_preview() {
-     	if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
-         	var final_index = complete_paste.length-1;
-          var flag_stop = true;
-      } else {
-          var final_index = start_index + char_to_display;
-      }
-
-      if (final_index != start_index){ // still have data to display
-          // Append the new content using text() and not append (XSS)
-          $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
-          start_index = final_index;
-          if (flag_stop)
-              nothing_to_display();
-      } else {
-          nothing_to_display();
-      }
-  }
-  // Update the modal when there is no more data
-  function nothing_to_display() {
-      var new_content = $(alert_message).hide();
-      $("#mymodalbody").find("#panel-body").append(new_content);
-      new_content.show('fast');
-      $("#load-more-button").hide();
-  }
-
-  function get_html_and_update_modal(event, truemodal) {
-      event.preventDefault();
-
-      var modal=truemodal;
-      var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
-      last_clicked_paste =  modal.attr('data-num');
-      $.get(url, function (data) {
-
-          // verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
-          var received_num = parseInt(data.split("|num|")[1]);
-          if (received_num == last_clicked_paste && can_change_modal_content) {
-              can_change_modal_content = false;
-
-              // clear data by removing html, body, head tags. prevent dark modal background stack bug.
-              var cleared_data = data.split("<body>")[1].split("</body>")[0];
-              $("#mymodalbody").html(cleared_data);
-
-              var button = $('<button type="button" id="load-more-button" class="btn btn-outline-primary rounded-circle px-1 py-0" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><i class="fas fa-arrow-circle-down mt-1"></i></button>');
-							button.tooltip(button);
-              $("#container-show-more").append(button);
-
-							$("#button_show_path").attr('href', '{{ url_for('objects_item.showItem')  }}?id=' + $(modal).attr('data-path'));
-              $("#button_show_path").show('fast');
-              $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-              if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                  nothing_to_display();
-              }
-							// collapse decoded
-							$('#collapseDecoded').collapse('hide');
-              // On click, donwload all paste's content
-              $("#load-more-button").on("click", function (event) {
-                  if (complete_paste == null) { //Donwload only once
-                      $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                          complete_paste = data;
-                          update_preview();
-                      });
-                  } else {
-                      update_preview();
-                  }
-              });
-          } else if (can_change_modal_content) {
-              $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
-          }
-      });
-  }
-
-
-</script>
-
-
-</html>

var/www/modules/Tags/templates/tagged.html

@@ -1,317 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-  <title>Tags - AIL</title>
-  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-  <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
-  <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
-
-  <style>
-      .tooltip-inner {
-         text-align: left;
-         height: 200%;
-         width: 200%;
-         max-width: 500px;
-         max-height: 500px;
-         font-size: 13px;
-      }
-      xmp {
-         white-space:pre-wrap;
-         word-wrap:break-word;
-      }
-      .test thead{
-      	background: #d91f2d;
-	      color: #fff;
-      }
-  </style>
-
-</head>
-<body>
-
-
-    {% include 'navbar.html' %}
-
-  <!-- Modal -->
-  <div id="mymodal" class="modal fade" role="dialog">
-   <div class="modal-dialog modal-lg">
-
-     <!-- Modal content-->
-     <div id="mymodalcontent" class="modal-content">
-       <div id="mymodalbody" class="modal-body" max-width="850px">
-         <p>Loading paste information...</p>
-         <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
-       </div>
-       <div class="modal-footer">
-         <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
-         <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-       </div>
-     </div>
-   </div>
-  </div>
-
-  <div id="page-wrapper">
-    <div class="row">
-      <div class="col-lg-12">
-        <h1 class="page-header" data-page="page-tags" >Tags</h1>
-      </div>
-      <!-- /.col-lg-12 -->
-    </div>
-    <!-- /.row -->
-      <div class="form-group input-group" >
-        <input id="ltags" style="width:100%;" type="text" name="ltags" autocomplete="off">
-
-          <div class="input-group-btn">
-            <button type="button" class="btn btn-search btn-primary btn-tags" onclick="searchTags()">
-                <span class="glyphicon glyphicon-search "></span>
-                <span class="label-icon">Search Tags</span>
-            </button>
-          </div>
-
-      </div>
-
-      <button class="btn btn-primary" onclick="emptyTags()" style="margin-bottom: 30px;">
-        <span class="glyphicon glyphicon-remove"></span>
-        <span class="label-icon">Clear Tags</span>
-      </button>
-
-    <table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_">
-      <thead>
-        <tr>
-          <th>#</th>
-          <th style="max-width: 800px;">Path</th>
-          <th>Date</th>
-          <th># of lines</th>
-          <th>Action</th>
-        </tr>
-      </thead>
-      <tbody>
-
-        {% for path in all_path %}
-          <tr>
-              <td> {{ loop.index0 }}</td>
-              <td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{path}}">{{ path }}</a>
-                <div>
-                  {% for tag in paste_tags[loop.index0] %}
-                  <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
-                    <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
-                  </a>
-                  {%  endfor %}
-                </div>
-              </td>
-              <td>{{ paste_date[loop.index0] }}</td>
-              <td>{{ paste_linenum[loop.index0] }}</td>
-              <td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span> <button type="button" class="btn-link" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('objects_item.showItem') }}?id={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
-          </tr>
-        {%  endfor %}
-
-      </tbody>
-    </table>
-
-    </br>
-    <div id="nbr_entry" class="alert alert-info">
-
-    </div>
-    <div id="div_stil_data">
-      <button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries(100)" style="display: True">Load 100 entries</button>
-      <button id="load_more_json_button2" type="button" class="btn btn-warning" onclick="add_entries(300)" style="display: True">Load 300 entries</button>
-      <img id="loading_gif_browse" src="{{url_for('static', filename='image/loading.gif') }}" heigt="20" width="20" style="margin: 2px;"></div>
-      </br>
-
-      <div>
-				<br></br>
-				<a class="btn btn-tags" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
-						<i class="fa fa-wrench fa-2x"></i>
-						<br></br>
-						<span class="label-icon">Edit Taxonomies List </span>
-				</a>
-				<a class="btn btn-tags" href="{{ url_for('Tags.galaxies') }}" target="_blank">
-						<i class="fa fa-rocket fa-2x"></i>
-						<br></br>
-						<span class="label-icon">Edit Galaxies List</span>
-				</a>
-			</div>
-
-      <div>
-				<br></br>
-				<a class="btn btn-tags" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
-						<i class="fa fa-cogs fa-2x"></i>
-						<br></br>
-						<span class="label-icon">MISP and Hive, auto push</span>
-				</a>
-			</div>
-
-  </div>
-
-</body>
-
-<script>
-var search_table;
-var ltags;
-var last_clicked_paste;
-var can_change_modal_content = true;
-$("#myTable_").attr('data-numElem', "{{ all_path|length }}");
-
-$(document).ready(function(){
-    search_table = $('#myTable_').DataTable({ "order": [[ 2, "desc" ]] });
-    $('[data-toggle="tooltip"]').tooltip();
-    $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
-        get_html_and_update_modal(event, $(this));
-    });
-
-
-    $("#load_more_json_button1").hide();
-    $("#load_more_json_button2").hide();
-    $("#nbr_entry").hide();
-    $("#loading_gif_browse").hide();
-
-    $.getJSON("{{ url_for('Tags.get_all_tags') }}",
-      function(data) {
-        activePage = "page-Tags"
-        $("#"+activePage).addClass("active");
-        var valueData = [
-          {% for tag in list_tag %}
-            '{{tag|safe}}',
-          {%  endfor %}
-        ];
-
-        ltags = $('#ltags').tagSuggest({
-          data: data,
-          value: valueData,
-          sortOrder: 'name',
-          maxDropHeight: 200,
-          name: 'ltags',
-          });
-
-      });
-
-});
-</script>
-
-<script>
-    function searchTags() {
-      var data = ltags.getValue();
-      window.location.replace("{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data);
-    }
-    function emptyTags() {
-      ltags.clear();
-    }
-</script>
-
-<!-- Dynamically update the modal -->
-<script>
-    // static data
-    var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
-    var complete_paste = null;
-    var char_to_display = {{ char_to_display }};
-    var start_index = 0;
-
-    // When the modal goes out, refresh it to normal content
-    $("#mymodal").on('hidden.bs.modal', function () {
-        can_change_modal_content = true;
-        $("#mymodalbody").html("<p>Loading paste information...</p>");
-        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
-        $("#mymodalbody").append(loading_gif); // Show the loading GIF
-        $("#button_show_path").attr('href', '');
-        $("#button_show_path").hide();
-        complete_paste = null;
-        start_index = 0;
-    });
-
-   // Update the paste preview in the modal
-   function update_preview() {
-       if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
-           var final_index = complete_paste.length-1;
-           var flag_stop = true;
-       } else {
-           var final_index = start_index + char_to_display;
-       }
-
-       if (final_index != start_index){ // still have data to display
-           // Append the new content using text() and not append (XSS)
-           $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
-           start_index = final_index;
-           if (flag_stop)
-               nothing_to_display();
-       } else {
-          nothing_to_display();
-      }
-   }
-   // Update the modal when there is no more data
-   function nothing_to_display() {
-       var new_content = $(alert_message).hide();
-       $("#mymodalbody").find("#panel-body").append(new_content);
-       new_content.show('fast');
-       $("#load-more-button").hide();
-   }
-
-   function get_html_and_update_modal(event, truemodal) {
-       event.preventDefault();
-
-       var modal=truemodal;
-       var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
-       last_clicked_paste =  modal.attr('data-num');
-       $.get(url, function (data) {
-
-           // verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
-           var received_num = parseInt(data.split("|num|")[1]);
-           if (received_num == last_clicked_paste && can_change_modal_content) {
-               can_change_modal_content = false;
-
-               // clear data by removing html, body, head tags. prevent dark modal background stack bug.
-               var cleared_data = data.split("<body>")[1].split("</body>")[0];
-               $("#mymodalbody").html(cleared_data);
-
-               var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-               button.tooltip();
-               $("#mymodalbody").children(".panel-default").append(button);
-
-               $("#button_show_path").attr('href', $(modal).attr('data-url'));
-               $("#button_show_path").show('fast');
-               $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-               if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                   nothing_to_display();
-               }
-               // On click, donwload all paste's content
-               $("#load-more-button").on("click", function (event) {
-                   if (complete_paste == null) { //Donwload only once
-                       $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                           complete_paste = data;
-                           update_preview();
-                       });
-                   } else {
-                       update_preview();
-                   }
-               });
-           } else if (can_change_modal_content) {
-               $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
-           }
-       });
-   }
-
-   // Use to bind the button with the new displayed data
-   // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
-
-   search_table.on( 'draw.dt', function () {
-       // Bind tooltip each time we draw a new page
-       $('[data-toggle="tooltip"]').tooltip();
-       // On click, get html content from url and update the corresponding modal
-       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
-           get_html_and_update_modal(event, $(this));
-       });
-   } );
-
-</script>

var/www/modules/dashboard/Flask_dashboard.py

@@ -28,7 +28,6 @@ import Flask_config
 app = Flask_config.app
 config_loader = Flask_config.config_loader
 baseUrl = Flask_config.baseUrl
-r_serv = Flask_config.r_serv
 r_serv_log = Flask_config.r_serv_log
 
 max_dashboard_logs = Flask_config.max_dashboard_logs
@@ -53,7 +52,7 @@ def event_stream():
         if msg['type'] == 'pmessage' and level != "DEBUG":
             yield 'data: %s\n\n' % json.dumps(msg)
 
-def get_queues(r):
+def get_queues():
     # We may want to put the llen in a pipeline to do only one query.
     return queues_modules.get_all_modules_queues_stats()
 
@@ -63,7 +62,7 @@ def get_date_range(date_from, num_day):
 
     for i in range(0, num_day+1):
         new_date = date.substract_day(i)
-        date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8])
+        date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}')
 
     return date_list
 
@@ -74,10 +73,10 @@ def dashboard_alert(log):
         utc_str = log[1:20]
         log = log[46:].split(';')
         if len(log) == 6:
-            time = datetime_from_utc_to_local(utc_str)
+            date_time = datetime_from_utc_to_local(utc_str)
             path = url_for('objects_item.showItem', id=log[5])
 
-            res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
+            res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
                    'paste': log[3], 'message': log[4], 'path': path}
             return res
         else:
@@ -116,7 +115,7 @@ def get_last_logs_json():
     date_range = get_date_range(date, max_day_search)
     while max_day_search != day_search and warning_found != warning_to_found:
 
-        filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log'
+        filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log'
         filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
 
         try:
@@ -147,7 +146,7 @@ def get_last_logs_json():
 @login_required
 @login_read_only
 def stuff():
-    return jsonify(row1=get_queues(r_serv))
+    return jsonify(row1=get_queues())
 
 
 # TODO: ADD UPDATE NOTE BY USER
@@ -169,9 +168,11 @@ def index():
         background_update = True
         update_message = ail_updates.get_update_background_message()
 
-    return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module,
+    return render_template("index.html", default_minute = default_minute,
+                           threshold_stucked_module=threshold_stucked_module,
                            log_select=log_select, selected=max_dashboard_logs,
                            background_update=background_update, update_message=update_message)
 
+
 # ========= REGISTRATION =========
 app.register_blueprint(dashboard, url_prefix=baseUrl)

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -17,7 +17,7 @@ from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
 from flask_login import login_required
 
 from Date import Date
-from HiddenServices import HiddenServices
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import crawlers
 
 # ============ VARIABLES ============
@@ -30,9 +30,6 @@ r_serv_onion = Flask_config.r_serv_onion
 r_serv_metadata = Flask_config.r_serv_metadata
 bootstrap_label = Flask_config.bootstrap_label
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
-import crawlers
-
 hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
 
 faup = Faup()

var/www/modules/rawSkeleton/Flask_rawSkeleton.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-'''
-    Flask functions and routes for the trending modules page
-'''
-import redis
-from flask import Flask, render_template, jsonify, request, Blueprint
-
-from Role_Manager import login_admin, login_analyst, login_read_only
-from flask_login import login_required
-
-# ============ VARIABLES ============
-import Flask_config
-
-app = Flask_config.app
-
-rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')
-
-# ============ FUNCTIONS ============
-def one():
-    return 1
-
-# ============= ROUTES ==============
-
-@rawSkeleton.route("/rawSkeleton/", methods=['GET'])
-@login_required
-@login_read_only
-def skeleton_page():
-    return render_template("rawSkeleton.html")
-
-
-# ========= REGISTRATION =========
-app.register_blueprint(rawSkeleton)

var/www/modules/rawSkeleton/templates/header_rawSkeleton.html

@@ -1 +0,0 @@
-<li id='page-rawSkeleton'><a href="{{ url_for('rawSkeleton.skeleton_page') }}"><i class="glyphicon glyphicon-new-window"></i> rawSkeleton page</a></li>

var/www/modules/rawSkeleton/templates/rawSkeleton.html

@@ -1,47 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-	<head>
-	  <meta charset="utf-8">
-	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-	  <title>Analysis Information Leak framework Dashboard</title>
-
-	  <!-- Core CSS -->
-	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
-	  <!-- JS -->
-	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-          <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-          <script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
-          <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-
-	</head>
-	<body>
-
-            {% include 'navbar.html' %}
-
-            <div id="page-wrapper">
-				<div class="row">
-				  <div class="col-lg-12">
-				  <h1 class="page-header" data-page="page-rawSkeleton" >rawSkeleton</h1>
-				  </div>
-				  <!-- /.col-lg-12 -->
-				</div>
-                <!-- /.row -->
-            </div>
-            <!-- /#page-wrapper -->
-
-            <script>
-                $(document).ready(function(){
-                    activePage = "page-rawSkeleton"
-                    $("#"+activePage).addClass("active");
-                });
-            </script>
-            <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-        </body>
-
-</html>

var/www/modules/search/templates/search.html

@@ -311,7 +311,7 @@
                    // On click, donwload all paste's content
                    $("#load-more-button").off('click.download').on("click.download", function (event) {
                        if (complete_paste == null) { //Donwload only once
-                           $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                           $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){
                                complete_paste = data;
                                update_preview();
                            });

var/www/modules/showpaste/Flask_showpaste.py

@@ -1,283 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-'''
-    Flask functions and routes for the trending modules page
-'''
-import json
-import os
-import sys
-import flask
-from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for, abort
-
-from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
-from flask_login import login_required
-
-import difflib
-
-import Paste
-import requests
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-import Item
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
-
-# ============ VARIABLES ============
-import Flask_config
-
-app = Flask_config.app
-baseUrl = Flask_config.baseUrl
-r_serv_metadata = Flask_config.r_serv_metadata
-r_serv_tags = Flask_config.r_serv_tags
-r_serv_statistics = Flask_config.r_serv_statistics
-max_preview_char = Flask_config.max_preview_char
-max_preview_modal = Flask_config.max_preview_modal
-DiffMaxLineLength = Flask_config.DiffMaxLineLength
-bootstrap_label = Flask_config.bootstrap_label
-misp_event_url = Flask_config.misp_event_url
-hive_case_url = Flask_config.hive_case_url
-vt_enabled = Flask_config.vt_enabled
-PASTES_FOLDER = Flask_config.PASTES_FOLDER
-SCREENSHOT_FOLDER = Flask_config.SCREENSHOT_FOLDER
-
-showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
-
-# ============ FUNCTIONS ============
-
-def get_item_screenshot_path(item):
-    screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot')
-    if screenshot:
-        screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
-    else:
-        screenshot = ''
-    return screenshot
-
-def get_item_basic_info(item):
-    item_basic_info = {}
-    item_basic_info['date'] = str(item.get_p_date())
-    item_basic_info['date'] = '{}/{}/{}'.format(item_basic_info['date'][0:4], item_basic_info['date'][4:6], item_basic_info['date'][6:8])
-    item_basic_info['source'] = item.get_item_source()
-    item_basic_info['size'] = item.get_item_size()
-
-    ## TODO: FIXME ##performance
-    item_basic_info['encoding'] = item._get_p_encoding()
-    ## TODO: FIXME ##performance
-    #item_basic_info['language'] = item._get_p_language()
-    ## TODO: FIXME ##performance
-    info_line = item.get_lines_info()
-    item_basic_info['nb_lines'] = info_line[0]
-    item_basic_info['max_length_line'] = info_line[1]
-
-    return item_basic_info
-
-def show_item_min(requested_path , content_range=0):
-    relative_path = None
-    if PASTES_FOLDER not in requested_path:
-        relative_path = requested_path
-        requested_path = os.path.join(PASTES_FOLDER, requested_path)
-    else:
-        relative_path = requested_path.replace(PASTES_FOLDER, '', 1)
-    # remove old full path
-    # requested_path = requested_path.replace(PASTES_FOLDER, '')
-    # escape directory transversal
-    if os.path.commonprefix((os.path.realpath(requested_path),PASTES_FOLDER)) != PASTES_FOLDER:
-        return 'path transversal detected'
-
-    item_info ={}
-
-    paste = Paste.Paste(requested_path)
-    item_basic_info = get_item_basic_info(paste)
-    item_info['nb_duplictates'] = paste.get_nb_duplicate()
-
-    ## TODO: use this for fix ?
-    item_content = paste.get_p_content()
-    char_to_display = len(item_content)
-    if content_range != 0:
-       item_content = item_content[0:content_range]
-
-    vt_enabled = Flask_config.vt_enabled
-
-
-    p_hashtype_list = []
-
-    print(requested_path)
-    l_tags = r_serv_metadata.smembers('tag:'+relative_path)
-    if relative_path is not None:
-        l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) )
-    item_info['tags'] = l_tags
-    item_info['name'] = relative_path.replace('/', ' / ')
-
-
-    l_64 = []
-    # load hash files
-    if r_serv_metadata.scard('hash_paste:'+relative_path) > 0:
-        set_b64 = r_serv_metadata.smembers('hash_paste:'+relative_path)
-        for hash in set_b64:
-            nb_in_file = r_serv_metadata.zscore('nb_seen_hash:'+hash, relative_path)
-            # item list not updated
-            if nb_in_file is None:
-                l_pastes = r_serv_metadata.zrange('nb_seen_hash:'+hash, 0, -1)
-                for paste_name in l_pastes:
-                    # dynamic update
-                    if PASTES_FOLDER in paste_name:
-                        score = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), paste_name)
-                        r_serv_metadata.zrem('nb_seen_hash:{}'.format(hash), paste_name)
-                        paste_name = paste_name.replace(PASTES_FOLDER, '', 1)
-                        r_serv_metadata.zadd('nb_seen_hash:{}'.format(hash), score, paste_name)
-                nb_in_file = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), relative_path)
-            nb_in_file = int(nb_in_file)
-            estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
-            file_type = estimated_type.split('/')[0]
-            # set file icon
-            if file_type == 'application':
-                file_icon = 'fa-file '
-            elif file_type == 'audio':
-                file_icon = 'fa-file-video '
-            elif file_type == 'image':
-                file_icon = 'fa-file-image'
-            elif file_type == 'text':
-                file_icon = 'fa-file-alt'
-            else:
-                file_icon =  'fa-file'
-            saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
-            if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
-                b64_vt = True
-                b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
-                b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
-            else:
-                b64_vt = False
-                b64_vt_link = ''
-                b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
-                # hash never refreshed
-                if b64_vt_report is None:
-                    b64_vt_report = ''
-
-            l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) )
-
-    crawler_metadata = {}
-    if 'infoleak:submission="crawler"' in l_tags:
-        crawler_metadata['get_metadata'] = True
-        crawler_metadata['domain'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'domain')
-        crawler_metadata['domain'] = crawler_metadata['domain'].rsplit(':', 1)[0]
-        crawler_metadata['paste_father'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'father')
-        crawler_metadata['real_link'] = r_serv_metadata.hget('paste_metadata:'+relative_path,'real_link')
-        crawler_metadata['screenshot'] = get_item_screenshot_path(relative_path)
-        # crawler_metadata['har_file'] = Item.get_item_har(relative_path)
-    else:
-        crawler_metadata['get_metadata'] = False
-
-    item_parent = Item.get_item_parent(requested_path)
-
-    misp_event = r_serv_metadata.get('misp_events:' + requested_path)
-    if misp_event is None:
-        misp_eventid = False
-        misp_url = ''
-    else:
-        misp_eventid = True
-        misp_url = misp_event_url + misp_event
-
-    hive_case = r_serv_metadata.get('hive_cases:' + requested_path)
-    if hive_case is None:
-        hive_caseid = False
-        hive_url = ''
-    else:
-        hive_caseid = True
-        hive_url = hive_case_url.replace('id_here', hive_case)
-
-    return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content,
-                            item_basic_info=item_basic_info, item_info=item_info,
-                            item_parent=item_parent,
-                            initsize=len(item_content),
-                            hashtype_list = p_hashtype_list,
-                            crawler_metadata=crawler_metadata,
-                            l_64=l_64, vt_enabled=vt_enabled, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
-
-# ============ ROUTES ============
-
-@showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab
-@login_required
-@login_read_only
-def showsaveditem_min():
-    requested_path = request.args.get('paste', '')
-    return show_item_min(requested_path)
-
-@showsavedpastes.route("/showsavedrawpaste/") #shows raw
-@login_required
-@login_read_only
-def showsavedrawpaste():
-    requested_path = request.args.get('paste', '')
-    paste = Paste.Paste(requested_path)
-    content = paste.get_p_content()
-    return Response(content, mimetype='text/plain')
-
-@showsavedpastes.route("/showpreviewpaste/")
-@login_required
-@login_read_only
-def showpreviewpaste():
-    num = request.args.get('num', '')
-    requested_path = request.args.get('paste', '')
-    return "|num|"+num+"|num|"+show_item_min(requested_path, content_range=max_preview_modal)
-
-
-@showsavedpastes.route("/getmoredata/")
-@login_required
-@login_read_only
-def getmoredata():
-    requested_path = request.args.get('paste', '')
-    paste = Paste.Paste(requested_path)
-    p_content = paste.get_p_content()
-    to_return = p_content[max_preview_modal-1:]
-    return to_return
-
-@showsavedpastes.route("/showDiff/")
-@login_required
-@login_analyst
-def showDiff():
-    s1 = request.args.get('s1', '')
-    s2 = request.args.get('s2', '')
-    p1 = Paste.Paste(s1)
-    p2 = Paste.Paste(s2)
-    maxLengthLine1 = p1.get_lines_info()[1]
-    maxLengthLine2 = p2.get_lines_info()[1]
-    if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength:
-        return "Can't make the difference as the lines are too long."
-    htmlD = difflib.HtmlDiff()
-    lines1 = p1.get_p_content().splitlines()
-    lines2 = p2.get_p_content().splitlines()
-    the_html = htmlD.make_file(lines1, lines2)
-    return the_html
-
-@showsavedpastes.route('/screenshot/<path:filename>')
-@login_required
-@login_read_only
-@no_cache
-def screenshot(filename):
-    return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)
-
-@showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
-@login_required
-@login_analyst
-def send_file_to_vt():
-    b64_path = request.form['b64_path']
-    paste = request.form['paste']
-    hash = request.form['hash']
-
-    ## TODO:  # FIXME:  path transversal
-    b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
-    b64_content = ''
-    with open(b64_full_path, 'rb') as f:
-        b64_content = f.read()
-
-    files = {'file': (hash, b64_content)}
-    response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=vt_auth)
-    json_response = response.json()
-    print(json_response)
-
-    vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
-    r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link)
-
-    return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste))
-
-# ========= REGISTRATION =========
-app.register_blueprint(showsavedpastes, url_prefix=baseUrl)

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -1,581 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <title>Paste information - AIL</title>
-  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-
-   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-   <link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
-
-   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-   <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-   <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-   <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
-
-  <style>
-    .scrollable-menu {
-      height: auto;
-      max-height: 200px;
-      overflow-x: hidden;
-      width:100%;
-    }
-
-		.red_table thead{
-			background: #d91f2d;
-			color: #fff;
-		}
-  </style>
-
-</head>
-<body>
-    <div class="panel panel-default">
-      <div class="panel-heading">
-        <h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
-        <h2 class="page-header" >
-          <div>
-
-            <div id="mymodal" class="modal fade" role="dialog">
-              <div class="modal-dialog modal-lg">
-
-                <!-- Modal content-->
-                <div id="mymodalcontent" class="modal-content">
-                  <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
-                    <p class="heading">Edit this tag</p>
-
-                  </div>
-
-                  <div class="modal-body">
-
-                    <div class="input-group" >
-                      <input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px" name="ltags">
-                    </div>
-
-                    <div class="btn-group btn-block">
-                        <button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Taxonomie Selection
-                          <i class="fa fa-chevron-down"></i>
-                        </button>
-                        <ul class="dropdown-menu scrollable-menu" role="menu">
-                            <li><a href="#" id="all-tags-taxonomies">All Tags <i class="fa fa-tags"></i></a></li>
-                            <li role="separator" class="divider"></li>
-                            {% for taxo in active_taxonomies %}
-                                <li><a href="#" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</a></li>
-                            {% endfor %}
-                        </ul>
-                    </div>
-                     <br/><br/>
-
-                    <div class="input-group">
-                      <input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px" name="ltagsgalaxies">
-                    </div>
-
-                    <div class="btn-group btn-block">
-                        <button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Galaxy Selection
-                          <i class="fa fa-chevron-down"></i>
-                        </button>
-                        <ul class="dropdown-menu scrollable-menu" role="menu">
-                            <li><a href="#" id="all-tags-galaxies">All Tags <i class="fa fa-tags"></i></a></li>
-                            <li role="separator" class="divider"></li>
-                            {% for galaxy in active_galaxies %}
-                                <li><a href="#" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</a></li>
-                            {% endfor %}
-                        </ul>
-                    </div>
-                  </div>
-
-                  <div class="modal-footer">
-                    <a class="btn btn-tags pull-left" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
-                        <span class="label-icon">Edit Taxonomies List </span>
-                        <i class="fa fa-wrench fa-2x"></i>
-                    </a>
-                    <a class="btn btn-tags pull-left" href="{{ url_for('Tags.galaxies') }}" target="_blank">
-                        <span class="label-icon">Edit Galaxies List</span>
-                        <i class="fa fa-rocket fa-2x"></i>
-                    </a>
-                    <button class="btn btn-primary btn-tags" onclick="addTags()">
-                        <span class="glyphicon glyphicon-plus"></span>
-                        <span class="label-icon">Add Tags</span>
-                    </button>
-                    <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                  </div>
-                </div>
-              </div>
-            </div>
-
-
-            {% for tag in list_tags %}
-            <span class="btn btn-{{ bootstrap_label[loop.index0 % 5] }} btn-lg pull-left" data-toggle="modal" data-target="#myModal_{{ loop.index0 }}">{{ tag[0] }}</span>
-
-              <!-- Modal edit this tag -->
-              <div class="modal fade" id="myModal_{{ loop.index0 }}" role="dialog">
-                <div class="modal-dialog">
-
-                  <div class="modal-content text-center">
-                    <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
-                      <p class="heading">Edit this tag</p>
-
-                    </div>
-                    <div class="modal-body">
-                      <span class="label label-{{ bootstrap_label[loop.index0 % 5] }}" >{{ tag[0] }}</span>
-                    </div>
-                    <div class="modal-footer center">
-
-                        {% if not tag[2] %}
-                          <a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=tp" class="btn btn-success pull-left" data-toggle="tooltip" title="Good Detection">
-                            <span class="glyphicon glyphicon-thumbs-up "></span>
-                          </a>
-                        {% endif %}
-                        {% if tag[2] %}
-                          <button class="btn btn-success pull-left disabled" data-toggle="tooltip" title="Good Detection">
-                            <span class="glyphicon glyphicon-thumbs-up "></span>
-                          </button>
-                        {% endif %}
-                        {% if not tag[3] %}
-                          <a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=fp" class="btn btn-danger pull-left" data-toggle="tooltip" title="Bad Detection">
-                            <span class="glyphicon glyphicon-thumbs-down "></span>
-                          </a>
-                        {% endif %}
-                        {% if tag[3] %}
-                          <button class="btn btn-danger pull-left disabled" data-toggle="tooltip" title="Bad Detection">
-                            <span class="glyphicon glyphicon-thumbs-down "></span>
-                          </button>
-                        {% endif %}
-
-                      {% if tag[1] %}
-                        <a href="{{ url_for('Tags.confirm_tag') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}" class="btn btn-primary">
-                          <span class="glyphicon glyphicon-ok "></span> Confirm this Tag
-                        </a>
-                      {% endif %}
-                      <a href="{{ url_for('Tags.remove_tag') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}" class="btn btn-danger">
-                        <span class="glyphicon glyphicon-trash "></span> Delete this Tag
-                      </a>
-                    </div>
-                  </div>
-
-                </div>
-              </div>
-
-            {%  endfor %}
-            <button type="button" class="btn btn-light btn-lg" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('Tags.taxonomies') }}">
-              <span class="glyphicon glyphicon-plus "></span>
-            </button>
-          </div>
-        </h2>
-
-        <table class="table table-condensed">
-          <thead>
-            <tr>
-              <th>Date</th>
-              <th>Source</th>
-              <th>Encoding</th>
-              <th>Language</th>
-              <th>Size (Kb)</th>
-              <th>Mime</th>
-              <th>Number of lines</th>
-              <th>Max line length</th>
-            </tr>
-          </thead>
-          <tbody>
-            <tr>
-            <td>{{ date }}</td>
-            <td>{{ source }}</td>
-            <td>{{ encoding }}</td>
-            <td>{{ language }}</td>
-            <td>{{ size }}</td>
-            <td>{{ mime }}</td>
-            <td>{{ lineinfo.0 }}</td>
-            <td>{{ lineinfo.1 }}</td>
-            </tr>
-          </tbody>
-        </table>
-
-        {% if item_parent %}
-          <div>
-            <a href="{{ url_for('showsavedpastes.showsavedpaste')}}?paste={{item_parent}}" target="_blank">{{item_parent}}</a>
-          </div>
-        {% endif %}
-
-        <div>
-          {% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%}
-            {% include 'import_export/block_add_user_object_to_export.html' %}
-          {% endwith %}
-
-          {% if hive %}
-          <button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
-            Create
-            <img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
-            Case
-          </button>
-          <!-- Modal HIVE-->
-          <div class="modal fade" id="myModal_hive" role="dialog">
-            <div class="modal-dialog">
-
-              <div class="modal-content">
-                <div class="modal-header text-center">
-                  <img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
-                </div>
-
-                <div class="modal-body">
-
-                  <form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
-
-                  <div class="input clear required">
-                    <label for="EventThreatLevelId">Threat Level</label>
-                    <select name="threat_level_hive" id="EventThreatLevelId" required="required">
-                      <option value="1">High</option>
-                      <option value="2" selected="selected">Medium</option>
-                      <option value="3">Low</option>
-                    </select>
-                  </div>
-                  <div class="input select required">
-                    <label for="TLP">TLP</label>
-                    <select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
-                      <option value="0">White</option>
-                      <option value="1">Green</option>
-                      <option value="2" selected="selected">Amber</option>
-                      <option value="3">Red</option>
-                    </select>
-                  </div>
-                  <div class="clear required">
-                    <label for="hive_case_title">Title</label>
-                    <input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
-                  </div>
-                  <div class="clear required">
-                    <label for="hive_description">Description</label>
-                    <input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
-                  </div>
-                  <input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
-
-                  {% if hive_caseid %}
-                  <br>
-                  <div class="list-group" id="misp_event">
-                    <li class="list-group-item active">The Hive Case already Created</li>
-                      <a target="_blank" href="{{ hive_url }}" class="list-group-item">{{ hive_url }}</a></div>
-                  {% endif %}
-
-                </div>
-                <div class="modal-footer center">
-                  <button href="" class="btn btn-primary">
-                    <span class="glyphicon glyphicon-ok "></span> Create Case
-                  </button>
-
-                </form>
-                  <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                </div>
-              </div>
-
-            </div>
-          </div>
-          {% endif %}
-        </div>
-
-        <div>
-					<a href="{{ url_for('correlation.show_correlation')}}?object_type=paste&correlation_id={{ request.args.get('paste') }}&correlation_objects=paste" target="_blank" style="font-size: 15px">
-						<button class="btn btn-info"><i class="fa fa-search"></i> Show Paste Correlation
-						</button>
-					</a>
-				</div>
-
-      </div>
-
-    <div class="panel-body" id="panel-body">
-
-      {% if  duplicate_list|length == 0 %}
-      {% else %}
-          <h3> Duplicate list: </h3>
-          <table id="tableDup" class="table table-striped table-bordered">
-          {% set i = 0 %}
-          <thead>
-          <tr>
-              <th>Hash type</th>
-              <th>Paste info</th>
-              <th>Date</th>
-              <th>Path</th>
-              <th>Action</th>
-          </tr>
-          </thead>
-          <tbody>
-          {% for dup_path in duplicate_list %}
-              <tr>
-                  <td>{{ hashtype_list[loop.index - 1] }}</td>
-                  <td>Similarity: {{ simil_list[loop.index - 1] }}%</td>
-                  <td>{{ date_list[loop.index - 1] }}</td>
-                  <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
-                  <td><a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{ request.args.get('paste') }}&s2={{ dup_path }}" class="fa fa-columns" title="Show differences"></a></td>
-              </tr>
-          {% endfor %}
-          </tbody>
-          </table>
-      {% endif %}
-
-      {% if  l_64|length != 0 %}
-          <h3> Hash files: </h3>
-          <table id="tableb64" class="red_table table table-striped table-bordered">
-          <thead>
-          <tr>
-              <th>estimated type</th>
-              <th>hash</th>
-              <th>saved_path</th>
-              <th>Virus Total</th>
-          </tr>
-          </thead>
-          <tbody>
-          {% for b64 in l_64 %}
-              <tr>
-                  <td><i class="fa {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
-                  <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a> ({{ b64[4] }})</td>
-                  <td>{{ b64[3] }}</td>
-                  <td style="text-align:center;">
-                    {% if  vt_enabled %}
-                      {% if not b64[5] %}
-                        <darkbutton_{{ b64[2] }}>
-                          <button id="submit_vt_{{ b64[2] }}" class="btn btn-primary" onclick="sendFileToVT('{{ b64[2] }}')">
-                            <i class="fa fa-paper-plane"></i>&nbsp;Send this file to VT
-                          </button>
-                        </darkbutton_{{ b64[2] }}>
-                      {% else %}
-                        <a class="btn btn-primary" target="_blank" href="{{ b64[6] }}"><i class="fa fa-link"> VT Report</i></a>
-                      {% endif %}
-                      <button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
-                        <div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span>&nbsp;{{ b64[7] }}</div>
-                      </button>
-                    {% else %}
-                      Virus Total submission is disabled
-                    {% endif %}
-                  </td>
-              </tr>
-          {% endfor %}
-          </tbody>
-          </table>
-      {% endif %}
-
-        {% if crawler_metadata['get_metadata'] %}
-          <div class="row">
-
-            <div class="col-md-5">
-              <div class="row">
-                <div class="panel panel-info">
-      						<div class="panel-heading">
-                    <i class="fa fa-eye-slash"></i> Crawled Paste
-      						</div>
-
-                      <table class="table table-hover table-striped">
-                        <tbody>
-                          <tr>
-                            <td>Domain</td>
-                            <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
-                          </tr>
-                          <tr>
-                            <td>Father</td>
-                            <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ crawler_metadata['paste_father'] }}" id='paste_father'>{{ crawler_metadata['paste_father'] }}</a></td>
-                          </tr>
-                          <tr>
-                            <td>Source link</td>
-                            <td>{{ crawler_metadata['real_link'] }}</td>
-                          </tr>
-                        </tbody>
-                      </table>
-
-                      {% if crawler_metadata['har_file'] %}
-                        button
-                      {% endif %}
-                </div>
-              </div>
-            </div>
-
-            <div class="col-md-7">
-              <div class="panel panel-info" style="text-align:center;">
-                <div class="panel-heading">
-                    <div class="row">
-                      <div class="col-md-8">
-                        <input class="center" id="blocks" type="range" min="1" max="50" value="{%if tags_safe%}13{%else%}0{%endif%}">
-                      </div>
-                      <div class="col-md-4">
-                        <button class="btn {%if tags_safe%}btn-primary{%else%}btn-danger{%endif%} btn-tags" onclick="blocks.value=50;pixelate();">
-                            <span class="glyphicon glyphicon-zoom-in"></span>
-                            <span class="label-icon">Full resolution</span>
-                        </button>
-                      </div>
-                    </div>
-                  </div>
-                </div>
-              <canvas id="canvas" style="width:100%;"></canvas>
-            </div>
-          </div>
-        {% endif %}
-
-        <h3> Content: </h3>
-        <a href="{{ url_for('showsavedpastes.showsavedrawpaste') }}?paste={{ request.args.get('paste') }}" id='raw_paste' > [Raw content] </a>
-        <p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
-    </div>
-  </div>
-
-</body>
-
-<script>
-    var ltags
-    var ltagsgalaxies
-    $(document).ready(function(){
-
-        $.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
-          function(data) {
-
-            ltags = $('#ltags').tagSuggest({
-              data: data,
-              maxDropHeight: 200,
-              name: 'ltags'
-            });
-          });
-
-        $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
-          function(data) {
-
-            ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
-              data: data,
-              maxDropHeight: 200,
-              name: 'ltagsgalaxies'
-            });
-          });
-
-        $('#tableDup').DataTable();
-        $('#tableb64').DataTable({
-  				"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
-  				"iDisplayLength": 5,
-  				"order": [[ 1, "asc" ]]
-  			});
-    });
-</script>
-
-<script>
-  function updateVTReport(hash) {
-    //updateReport
-    $.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
-      function(data) {
-        content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
-        $( "#report_vt_"+hash ).html(content);
-    });
-  }
-
-  function sendFileToVT(hash) {
-    //send file to vt
-    $.getJSON("{{ url_for('hashDecoded.send_file_to_vt_js') }}?hash="+hash,
-      function(data) {
-        var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
-        $('#submit_vt_'+hash).remove();
-        $('darkbutton_'+hash).append(content);
-    });
-  }
-
-</script>
-
-<script>
-    jQuery("#all-tags-taxonomies").click(function(e){
-      //change input tags list
-      $.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
-        function(data) {
-          ltags.setData(data)
-      });
-    });
-
-</script>
-<script>
-    jQuery("#all-tags-galaxies").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
-        function(data) {
-          ltagsgalaxies.setData(data)
-      });
-    });
-
-</script>
-
-<script>
-    function addTags() {
-      var tags = ltags.getValue()
-      var tagsgalaxy = ltagsgalaxies.getValue()
-      var path = '{{ request.args.get('paste') }}'
-      window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id=" + path + "&object_type=item");
-    }
-
-</script>
-
-<script>
-{% for taxo in active_taxonomies %}
-    jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
-        function(data) {
-          ltags.setData(data)
-  });
-});
-{% endfor %}
-</script>
-<script>
-{% for galaxy in active_galaxies %}
-    jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
-        function(data) {
-          ltagsgalaxies.setData(data)
-  });
-});
-{% endfor %}
-</script>
-
-{% if crawler_metadata['get_metadata'] %}
-  <script>
-  var ctx = canvas.getContext('2d'), img = new Image();
-
-  /// turn off image smoothing
-  ctx.webkitImageSmoothingEnabled = false;
-  ctx.imageSmoothingEnabled = false;
-
-  img.onload = pixelate;
-  img.addEventListener("error", img_error);
-  var draw_img = false;
-
-  img.src = "{{ url_for('showsavedpastes.screenshot', filename=crawler_metadata['screenshot']) }}";
-
-  function pixelate() {
-
-    /// use slider value
-    if( blocks.value == 50 ){
-        size = 1;
-    } else {
-        var size = (blocks.value) * 0.01;
-    }
-
-    canvas.width = img.width;
-    canvas.height = img.height;
-
-    /// cache scaled width and height
-    w = canvas.width * size;
-    h = canvas.height * size;
-
-    /// draw original image to the scaled size
-    ctx.drawImage(img, 0, 0, w, h);
-
-    /// pixelated
-    ctx.drawImage(canvas, 0, 0, w, h, 0, 0, canvas.width, canvas.height);
-
-  }
-
-  function img_error() {
-    img.onerror=null;
-    img.src="{{ url_for('static', filename='image/AIL.png') }}";
-    blocks.value = 50;
-    pixelate;
-  }
-
-  blocks.addEventListener('change', pixelate, false);
-  </script>
-{% endif %}
-
-</html>

var/www/templates/correlation/metadata_card_cryptocurrency.html

@@ -22,7 +22,7 @@
 										<svg height="26" width="26">
 											<g class="nodes">
 												<circle cx="13" cy="13" r="13" fill="orange"></circle>
-												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 											</g>
 										</svg>
 										{{ dict_object["metadata"]["type_id"] }}

var/www/templates/correlation/metadata_card_cve.html

@@ -30,7 +30,7 @@
 										<svg height="26" width="26">
 											<g class="nodes">
 												<circle cx="13" cy="13" r="13" fill="orange"></circle>
-												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 											</g>
 										</svg>
 									</td>

var/www/templates/correlation/metadata_card_decoded.html

@@ -32,7 +32,7 @@
 										<svg height="26" width="26">
 											<g class="nodes">
 												<circle cx="13" cy="13" r="13" fill="orange"></circle>
-												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 											</g>
 										</svg>
 										{{ dict_object["metadata"]["file_type"] }}
@@ -56,7 +56,7 @@
 					Tags:
 					{% for tag in dict_object["metadata"]['tags'] %}
 						<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
-						data-tagid="{{ tag }}" data-objtype="decoded" data-objid="{{ dict_object["correlation_id"] }}">
+						data-tagid="{{ tag }}" data-objtype="decoded" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
 							{{ tag }}
 						</button>
 					{%  endfor %}

var/www/templates/correlation/metadata_card_domain.html

@@ -21,7 +21,7 @@
 								<svg height="26" width="26">
 									<g class="nodes">
 										<circle cx="13" cy="13" r="13" fill="orange"></circle>
-										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 									</g>
 								</svg>
 								{{ dict_object["metadata"]["type_id"] }}

var/www/templates/correlation/metadata_card_pgp.html

@@ -22,7 +22,7 @@
 										<svg height="26" width="26">
 											<g class="nodes">
 												<circle cx="13" cy="13" r="13" fill="orange"></circle>
-												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 											</g>
 										</svg>
 										{{ dict_object["metadata"]["type_id"] }}
@@ -39,6 +39,16 @@
 					</div>
 				</div>
 			</li>
+            {% if dict_object["metadata"]['tags'] %}
+                <li class="list-group-item py-0">
+				    <div class="my-3">
+				    	<b>Tags:</b>
+				    	{% for tag in dict_object["metadata"]['tags'] %}
+				    		<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+				    	{%  endfor %}
+				    </div>
+			</li>
+            {% endif %}
 		</ul>
 
 		{% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}

var/www/templates/correlation/metadata_card_screenshot.html

@@ -68,7 +68,7 @@
 					Tags:
 					{% for tag in dict_object["metadata"]['tags'] %}
 						<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
-						data-tagid="{{ tag }}" data-objtype="image" data-objid="{{ dict_object["correlation_id"] }}">
+						data-tagid="{{ tag }}" data-objtype="screenshot" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
 							{{ tag }}
 						</button>
 					{%  endfor %}
@@ -99,7 +99,7 @@ img.onload = pixelate;
 img.addEventListener("error", img_error);
 var draw_img = false;
 
-img.src = "{{ url_for('showsavedpastes.screenshot', filename=dict_object['metadata']['img']) }}";
+img.src = "{{ url_for('objects_item.screenshot', filename=dict_object['metadata']['img']) }}";
 
 function pixelate() {
   /// use slider value

var/www/templates/correlation/metadata_card_username.html

@@ -22,7 +22,7 @@
 										<svg height="26" width="26">
 											<g class="nodes">
 												<circle cx="13" cy="13" r="13" fill="orange"></circle>
-												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
 											</g>
 										</svg>
 										{{ dict_object["metadata"]["type_id"] }}

var/www/templates/correlation/show_correlation.html

@@ -511,7 +511,7 @@ if (d.popover) {
 					desc = desc + "</dl>"
 
 					if (data["img"]) {
-						desc = desc + "<img src={{ url_for('showsavedpastes.screenshot', filename="") }}" + data["img"] +" class=\"img-thumbnail\" />";
+						desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}" + data["img"] +" class=\"img-thumbnail\" />";
 					}
 
 					if (data["tags"]) {

var/www/templates/crawler/crawler_splash/domain_explorer.html

@@ -11,8 +11,8 @@
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
     <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
     <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
-  <script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
- 	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+ 	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
 
     <style>
         .card-columns {
@@ -217,7 +217,7 @@ blocks.addEventListener('change', pixelate_all, false);
 {% for dict_domain in dict_data['list_elem'] %}
   {% if 'screenshot' in dict_domain %}
     {% if dict_domain['is_tags_safe'] %}
-      var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
+      var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}";
     {% else %}
     var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}";
     {% endif %}

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -85,7 +85,7 @@
 								<div>
                                     {% include 'modals/edit_tag.html' %}
 									{% for tag in dict_domain['tags'] %}
-                                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" data-tagid="{{ tag }}" data-objtype="domain" data-objid="{{ dict_domain['domain'] }}">
+                                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" data-tagid="{{ tag }}" data-objtype="domain" data-objsubtype="" data-objid="{{ dict_domain['domain'] }}">
                                             {{ tag }}
                                         </button>
 									{%  endfor %}
@@ -526,7 +526,7 @@ function toggle_sidebar(){
 
 <script>
 var ctx = canvas.getContext('2d'), img = new Image();
-var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}";
+var base_url = "{{ url_for('objects_item.screenshot', filename="") }}";
 var screenshot_href = "{{ url_for('objects_item.showItem') }}?id=";
 
 /// turn off image smoothing

var/www/templates/domains/domains_result_list.html

@@ -174,7 +174,7 @@ blocks.addEventListener('change', pixelate_all, false);
 {% for dict_domain in l_dict_domains['list_elem'] %}
   {% if 'screenshot' in dict_domain %}
     {% if dict_domain['is_tags_safe'] %}
-      var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
+      var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}";
     {% else %}
     var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}";
     {% endif %}

var/www/templates/import_export/import_object.html

@@ -84,8 +84,8 @@
 											<div class="my-1">
 												<svg height="26" width="26">
 													<g class="nodes">
-														<circle cx="13" cy="13" r="13" fill="{{ dict_obj['node']['node_color'] }}"></circle>
+														<circle cx="13" cy="13" r="13" fill="{{ dict_obj['node']['color'] }}"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_obj['node']['icon_class'] }}" font-size="16px">{{ dict_obj['node']['icon_text'] }}</text>
+														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_obj['node']['style'] }}" font-size="16px">{{ dict_obj['node']['icon'] }}</text>
 													</g>
 												</svg>
 											</div>

var/www/templates/modals/add_tags.html

@@ -126,6 +126,6 @@ jQuery("#all-tags-galaxies").click(function(e){
 function addTags() {
   var tags = ltags.getValue()
   var tagsgalaxy = ltagsgalaxies.getValue()
-  window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}");
+  window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&type={{ modal_add_tags['object_type'] }}&subtype={{ modal_add_tags['object_subtype'] }}&id={{ modal_add_tags['object_id'] }}");
 }
 </script>

var/www/templates/modals/edit_tag.html

@@ -15,17 +15,17 @@
 
       <div class="modal-footer">
 
-        <!-- <a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=tp" class="btn btn-outline-success mr-0" data-toggle="tooltip" title="Good Detection">
+        <!-- <a href="?object_id=rrrrr&tag=eeeeee&status=tp" class="btn btn-outline-success mr-0" data-toggle="tooltip" title="Good Detection">
           <i class="fas fa-thumbs-up"></i>
         </a>
-        <a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=fp" class="btn btn-outline-danger mr-auto" data-toggle="tooltip" title="Bad Detection">
+        <a href="?object_id=rrrrr&tag=eeeeee&status=fp" class="btn btn-outline-danger mr-auto" data-toggle="tooltip" title="Bad Detection">
           <i class="fas fa-thumbs-down"></i>
-        </a>
-
-        <a href="{{ url_for('Tags.confirm_tag') }}?object_id=rrrrr&tag=eeeeee" class="btn btn-primary">
-          <i class="fas fa-check"></i> Confirm this Tag
         </a> -->
 
+        <a href="#" class="btn btn-primary" id="modal_tag_confirm">
+          <i class="fas fa-check"></i> Confirm this Tag
+        </a>
+
         <a href="#" class="btn btn-danger" id="modal_tag_edit_delete_tag">
           <i class="fas fa-trash-alt"></i> Delete this Tag
         </a>
@@ -38,12 +38,19 @@
 <script>
 // tagid + objtype + objid
 $('#edit_tags_modal').on('show.bs.modal', function (event) {
+    var tag_confirm = $('#modal_tag_confirm');
+    tag_confirm.hide();
     var button = $(event.relatedTarget);
     var tagid = button.data('tagid')
     var objtype = button.data('objtype')
+    var objsubtype = button.data('objsubtype')
     var objid = button.data('objid')
     var modal = $(this)
     modal.find('#modal_tag_edit_tag_id').text(tagid)
-    modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
+    if (tagid.startsWith("infoleak:automatic-detection")) {
+        tag_confirm.show();
+        modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid);
+    }
+    modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid);
   })
 </script>

var/www/templates/modals/show_min_item.html

@@ -69,13 +69,10 @@ function get_html_and_update_modal(event, truemodal) {
     event.preventDefault();
 
     var modal=truemodal;
-    var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
+    var url = " {{ url_for('objects_item.item_preview')  }}?id=" + modal.attr('data-path');
-    last_clicked_paste =  modal.attr('data-num');
     $.get(url, function (data) {
 
         // verify that the reveived data is really the current clicked item. Otherwise, ignore it.
-        var received_num = parseInt(data.split("|num|")[1]);
-        if (received_num == last_clicked_paste && can_change_modal_content) {
             can_change_modal_content = false;
 
             // clear data by removing html, body, head tags. prevent dark modal background stack bug.
@@ -97,7 +94,7 @@ function get_html_and_update_modal(event, truemodal) {
             // On click, donwload all item's content
             $("#load-more-button").on("click", function (event) {
                 if (complete_item == null) { //Donwload only once
-                    $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                    $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){
                         complete_item = data;
                         update_preview();
                     });
@@ -105,9 +102,6 @@ function get_html_and_update_modal(event, truemodal) {
                     update_preview();
                 }
             });
-        } else if (can_change_modal_content) {
-            $("#modal_show_min_item_body").html("Ignoring previous not finished query of item #" + received_num);
-        }
     });
 }
 

var/www/templates/objects/item/show_item.html

@@ -66,11 +66,11 @@
 
       <div>
         <h5>
-          <div>
+          <span>
             {% include 'modals/edit_tag.html' %}
             {% for tag in meta['tags'] %}
               <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
-              data-tagid="{{ tag }}" data-objtype="item" data-objid="{{ meta['id'] }}">
+              data-tagid="{{ tag }}" data-objtype="item" data-objsubtype="" data-objid="{{ meta['id'] }}">
                 {{ tag }}
               </button>
 
@@ -80,7 +80,7 @@
             <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
               <i class="far fa-plus-square"></i>
             </button>
-          </div>
+          </span>
         </h5>
       </div>
 
@@ -200,7 +200,7 @@
                       </a>
                     </td>
                     <td>
-                      <a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{meta['id']}}&s2={{duplicate_id}}" class="fa fa-columns" title="Show diff"></a>
+                      <a target="_blank" href="{{ url_for('objects_item.object_item_diff') }}?id1={{meta['id']}}&id2={{duplicate_id}}" class="fa fa-columns" title="Show diff"></a>
                     </td>
                   </tr>
                 {% endfor %}
@@ -362,7 +362,7 @@
           <a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">Others</a>
           <div class="dropdown-menu">
             <a class="dropdown-item" href="{{ url_for('objects_item.item_raw_content', id=meta['id']) }}"><i class="far fa-file"></i> &nbsp;Raw Content</a>
-            <a class="dropdown-item" href="{{ url_for('objects_item.item_download', id=meta['id']) }}"><i class="fas fa-download"></i> &nbsp;Download</i></a>
+            <a class="dropdown-item" href="{{ url_for('objects_item.item_download', id=meta['id']) }}"><i class="fas fa-download"></i> &nbsp;Download</a>
           </div>
         </li>
       </ul>
@@ -416,7 +416,11 @@
   img.addEventListener("error", img_error);
   var draw_img = false;
 
-  img.src = "{{ url_for('showsavedpastes.screenshot', filename=meta['crawler']['screenshot']) }}";
+  {% if meta['crawler']['screenshot'] %}
+      img.src = "{{ url_for('objects_item.screenshot', filename=meta['crawler']['screenshot']) }}";
+  {% else %}
+      img.src = "";
+  {% endif %}
 
   function pixelate() {
 

var/www/templates/objects/item/show_item_min.html

@@ -10,7 +10,7 @@
     <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
     <link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
 
-  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
@@ -34,7 +34,7 @@
 
   <div class="card mb-2">
     <div class="card-header bg-dark">
-      <h3 class="text-white text-center" >{{ item_info['name'] }}</h3>
+      <h3 class="text-white text-center" >{{ meta['id'] }}</h3>
     </div>
     <div class="card-body pb-1">
       <table class="table table-condensed table-responsive">
@@ -50,19 +50,19 @@
         </thead>
         <tbody>
           <tr>
-          <td>{{ item_basic_info['date'] }}</td>
+          <td>{{ meta['date'] }}</td>
-          <td>{{ item_basic_info['source'] }}</td>
+          <td>{{ meta['source'] }}</td>
-          <td>{{ item_basic_info['encoding'] }}</td>
+          <td>{{ meta['mimetype'] }}</td>
-          <td>{{ item_basic_info['size'] }}</td>
+          <td>{{ meta['size'] }}</td>
-          <td>{{ item_basic_info['nb_lines'] }}</td>
+          <td>{{ meta['lines']['nb'] }}</td>
-          <td>{{ item_basic_info['max_length_line'] }}</td>
+          <td>{{ meta['lines']['max_length'] }}</td>
           </tr>
         </tbody>
       </table>
 
       <div>
         <h5>
-          {% for tag in item_info['tags'] %}
+          {% for tag in meta['tags'] %}
             <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
           {%  endfor %}
         </h5>
@@ -71,9 +71,9 @@
     </div>
   </div>
 
-  {% if item_parent %}
+  {% if meta['parent'] %}
     <div class="list-group" id="item_parent">
-      <a href="{{ url_for('objects_item.showItem')}}?id={{item_parent}}" target="_blank">{{item_parent}}</a>
+    parent: <a href="{{ url_for('objects_item.showItem')}}?id={{meta['parent']}}" target="_blank">{{meta['parent']}}</a>
     </div>
   {% endif %}
 
@@ -91,13 +91,13 @@
     </div>
   {% endif %}
 
-  {% if  item_info['nb_duplictates'] != 0 %}
+  {% if  meta['duplicates'] != 0 %}
     <div id="accordionDuplicate" class="mb-2">
       <div class="card">
         <div class="card-header py-1" id="headingDuplicate">
           <div class="my-1">
             <i class="far fa-clone"></i> duplicates&nbsp;&nbsp;
-            <div class="badge badge-warning">{{item_info['nb_duplictates']}}</div>
+            <div class="badge badge-warning">{{meta['duplicates']}}</div>
           </div>
         </div>
       </div>
@@ -105,52 +105,43 @@
   {% endif %}
 
 
-  {% if  l_64|length != 0 %}
+  {% if  meta['nb_correlations'] %}
-  <div id="accordionDecoded" class="mb-3">
+    <div id="accordionCorrelation" class="mb-2">
       <div class="card">
-      <div class="card-header py-1" id="headingDecoded">
+        <div class="card-header py-1" id="headingCorrelation">
-        <div class="row">
+          <div class="my-1">
-          <div class="col-11">
+              <h5>
-            <div class="mt-2">
+                  {% for obj_type in meta['nb_correlations'] %}
-              <i class="fas fa-lock-open"></i> Decoded Files&nbsp;&nbsp;
+                      {% if meta['nb_correlations'][obj_type] %}
-              <div class="badge badge-warning">{{l_64|length}}</div>
+                          <span class="badge badge-dark">
-            </div>
+                              {% if obj_type=='cve' %}
-          </div>
+                                  <i class="fas fa-bug"></i>
-          <div class="col-1">
+                              {% elif obj_type=='cryptocurrency' %}
-            <button class="btn btn-link py-2 rotate" data-toggle="collapse" data-target="#collapseDecoded" aria-expanded="true" aria-controls="collapseDecoded">
+                                  <i class="fas fa-coins"></i>
-              <i class="fas fa-chevron-circle-down"></i>
+                              {% elif obj_type=='decoded' %}
-            </button>
+                                  <i class="fas fa-lock-open"></i>
-          </div>
+                              {% elif obj_type=='pgp' %}
-        </div>
+                                  <i class="fas fa-key"></i>
-      </div>
+                              {% elif obj_type=='username' %}
-
+                                  <i class="fas fa-user"></i>
-      <div id="collapseDecoded" class="collapse show" aria-labelledby="headingDecoded" data-parent="#accordionDecoded">
+                              {% elif obj_type=='screenshot' %}
-        <div class="card-body">
+                                  <i class="fas fa-image"></i>
-
+                              {% else %}
-          <table id="tableb64" class="red_table table table-striped">
+                                  <i class="fas fa-cube"></i>
-            <thead>
+                              {% endif %}
-              <tr>
+                              {{ obj_type }}&nbsp;
-                <th>estimated type</th>
+                              <div class="badge badge-warning">{{meta['nb_correlations'][obj_type]}}</div>
-                <th>hash</th>
+                          </span>
-              </tr>
+                      {% endif %}
-            </thead>
-            <tbody>
-              {% for b64 in l_64 %}
-                <tr>
-                  <td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
-                  <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste"</a> {{b64[2]}} ({{ b64[4] }})</td>
-                </tr>
                   {% endfor %}
-            </tbody>
+              </h5>
-          </table>
-
           </div>
         </div>
       </div>
     </div>
   {% endif %}
 
-  {% if crawler_metadata['get_metadata'] %}
+  {% if 'crawler' in meta %}
   <div id="accordionCrawler" class="mb-3">
     <div class="card">
       <div class="card-header py-1" id="headingCrawled">
@@ -175,15 +166,15 @@
             <tbody>
               <tr>
                 <td>Domain</td>
-                <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
+                <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ meta['crawler']['domain'] }}" id='domain'>{{ meta['crawler']['domain'] }}</a></td>
               </tr>
               <tr>
                 <td>Father</td>
-                <td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{ crawler_metadata['paste_father'] }}" id='paste_father'>{{ crawler_metadata['paste_father'] }}</a></td>
+                <td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{ meta['parent'] }}" id='paste_father'>{{ meta['parent'] }}</a></td>
               </tr>
               <tr>
                 <td>Url</td>
-                <td>{{ crawler_metadata['real_link'] }}</td>
+                <td>{{ meta['crawler']['url'] }}</td>
               </tr>
             </tbody>
           </table>
@@ -229,7 +220,7 @@
   </div>
 
 
-  <p class="my-0" data-initsize="{{ initsize }}"> <pre id="paste-holder" class="border">{{ content }}</pre></p>
+  <p class="my-0" data-initsize="{{ initsize }}"> <pre id="paste-holder" class="border">{{ meta['content'] }}</pre></p>
 
   <script>
       var ltags
@@ -247,7 +238,7 @@
       });
   </script>
 
-{% if crawler_metadata['get_metadata'] %}
+{% if 'crawler' in meta %}
   <script>
   var ctx = canvas.getContext('2d'), img = new Image();
 
@@ -259,7 +250,7 @@
   img.addEventListener("error", img_error);
   var draw_img = false;
 
-  img.src = "{{ url_for('showsavedpastes.screenshot', filename=crawler_metadata['screenshot']) }}";
+  img.src = "{{ url_for('objects_item.screenshot', filename=meta['crawler']['screenshot']) }}";
 
   function pixelate() {
 

var/www/templates/tags/galaxies.html

@@ -0,0 +1,123 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Galaxies - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <h3 class="mt-2 text-secondary">
+					<i class="fas fa-rocket"></i> Galaxies:
+				</h3>
+
+                <table class="table table-bordered table-striped table-hover" id="myTable_">
+                    <thead class="thead-dark">
+                        <tr>
+                            <th>Name</th>
+                            <th>Description</th>
+                            <th>Namespace</th>
+                            <th>Enabled</th>
+                            <th>Active Tags</th>
+                            <th></th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for galaxy in galaxies %}
+                            <tr>
+                                <td><i class="{{ galaxy['icon'] }}"></i> {{ galaxy['name'] }}</td>
+                                <td>{{ galaxy['description'] }}</td>
+                                <td>{{ galaxy['namespace'] }}</td>
+                                <td class="text-center">
+                                    {% if galaxy['enebled'] %}
+                                        <div style="color:Green;">
+                                            <i class="fas fa-check-circle fa-2x"></i>
+										</div>
+                                    {% else %}
+                                        <div style="color:Red;">
+											<i class="fas fa-times-circle fa-2x"></i>
+										</div>
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    <h5>
+                                        <span class="badge badge-dark">
+                                            <b>{{ galaxy['nb_active_tags'] }}</b>/{{ galaxy['nb_tags'] }}
+                                        </span>
+                                    </h5>
+                                </td>
+                                <td>
+                                    <a class="btn btn-outline-secondary" href="{{ url_for('tags_ui.tags_galaxy') }}?galaxy={{ galaxy['type'] }}">
+                                        <i class="fas fa-tag"></i>
+                                    </a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+
+                </table>
+
+                <br>
+                <br>
+                <br>
+                <br>
+
+    	    </div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+
+$(document).ready(function(){
+  $("#navbar-tags").addClass("active");
+  $("#nav_tags_config").removeClass("text-muted");
+  $("#nav_galaxies").addClass("active");
+
+
+  $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] });
+
+});
+</script>
+
+<script>
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</html>

var/www/templates/tags/galaxy.html

@@ -0,0 +1,219 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Galaxy - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="my-3">
+                    <a href="{{ url_for('tags_ui.tags_galaxies') }}" class="pull-left">
+                        <i class="fa fa-arrow-left fa"></i> List Galaxies
+                    </a>
+                </div>
+
+                <div class="card">
+                    <div class="card-header bg-secondary text-white">
+                        <div class="d-flex">
+                            <span class="h3">{{ galaxy['name'] }}</span>
+
+                            <span class="h5 ml-auto">
+                                {% if galaxy['enabled'] %}
+                                    <span class="badge badge-success">Enabled</span>
+                                {% else %}
+                                    <span class="badge badge-danger">Disabled</span>
+                                {% endif %}
+                            </span>
+                        </div>
+                    </div>
+
+                    <div class="card-body">
+                        <table class="table table-sm table-hover">
+                            <tbody>
+                                <tr>
+                                    <td><b>Description</b></td>
+                                    <td>{{ galaxy['description'] }}</td>
+                                </tr>
+                                {% if galaxy['namespace'] %}
+                                    <tr>
+                                        <td><b>Namespace</b></td>
+                                        <td>{{ galaxy['namespace'] }}</td>
+                                    </tr>
+                                {% endif %}
+                                {% if galaxy['category'] %}
+                                    <tr>
+                                        <td><b>Category</b></td>
+                                        <td>{{ galaxy['category'] }}</td>
+                                    </tr>
+                                {% endif %}
+                                {% if galaxy['type'] %}
+                                    <tr>
+                                        <td><b>Type</b></td>
+                                        <td>{{ galaxy['type'] }}</td>
+                                    </tr>
+                                {% endif %}
+                                {% if galaxy['source'] %}
+                                    <tr>
+                                        <td><b>Source</b></td>
+                                        <td>{{ galaxy['source'] }}</td>
+                                    </tr>
+                                {% endif %}
+                                <tr>
+                                    <td><b>Version</b></td>
+                                    <td>{{ galaxy['version'] }}</td>
+                                </tr>
+                            </tbody>
+                        </table>
+
+                        <div class="mt-4">
+                            {% if galaxy['enabled'] %}
+                                <a href="{{ url_for('tags_ui.galaxy_disable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-danger">
+                                    <i class="fas fa-times"></i> Disable Galaxy
+                                </a>
+                            {% else %}
+                                <a href="{{ url_for('tags_ui.galaxy_enable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-success">
+                                    <i class="fas fa-check"></i> Enable Galaxy
+                                </a>
+                            {% endif %}
+                        </div>
+                    </div>
+                </div>
+
+                <form action="{{ url_for('tags_ui.galaxy_enable_tags') }}" id="checkboxForm">
+				    <input type="hidden" value="{{ galaxy['type'] }}" name="galaxy">
+
+                    <table class="table table-bordered table-hover my-4">
+                        <thead class="bg-dark text-white">
+                            <tr>
+                                <th></th>
+                                <th>Tag</th>
+                                <th>Description</th>
+                                <th class="text-center">Enabled</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for tag in galaxy['tags'] %}
+                                <tr>
+                                    <td class="text-center">
+                                        <div class="custom-control custom-switch">
+                                            <input class="custom-control-input" type="checkbox" name="tags" id="{{ tag['tag'] }}" value="{{ tag['tag'] }}" {% if tag['enabled'] %}checked{% endif %}>
+                                            <label class="custom-control-label" for="{{ tag['tag'] }}"></label>
+                                        </div>
+                                    </td>
+                                    <td>
+                                        <a href="{{ url_for('tags_ui.tags_galaxy_tag') }}?galaxy={{ galaxy['type'] }}&tag={{ tag['tag'] }}" class="badge badge-info">{{ tag['tag'] }}</a>
+                                        {% if tag['synonyms'] %}
+                                            <div>
+                                                Synonym: <i>
+                                                {% for synonym in tag['synonyms'] %}
+                                                    {{ synonym }},
+                                                {% endfor %}
+                                                </i>
+                                            </div>
+                                        {% endif %}
+                                    </td>
+                                    <td>
+                                        {% if 'description' in tag %}
+                                            {{ tag['description'] }}
+                                        {% endif %}
+                                    </td>
+                                    <td>
+                                        <div class="text-center">
+                                            {% if tag['enabled'] %}
+                                                <div style="color:Green;">
+                                                    <i class="fas fa-check-circle"></i>
+                                                </div>
+                                            {% else %}
+                                                <div style="color:Red;">
+                                                    <i class="fas fa-times-circle"></i>
+                                                </div>
+                                            {% endif %}
+                                        </div>
+                                    </td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+
+                    <div class="input-group-btn">
+                        <button class="btn btn-primary btn-lg" onclick="submitEnabledTags()">
+                            <i class="fas fa-check-square"></i> Update Tags
+                        </button>
+                    </div>
+
+                </form>
+
+                <br><br><br><br>
+
+    	    </div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+    $("#navbar-tags").addClass("active");
+    $("#nav_tags_config").removeClass("text-muted");
+
+    table = $('#myTable_').DataTable(
+        {
+            "aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        }
+    );
+
+});
+</script>
+
+<script>
+function submitEnabledTags() {
+    table.destroy();
+	table = $('#myTable_').DataTable(
+		{
+			"iDisplayLength": -1,
+		}
+	);
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+
+</html>

var/www/templates/tags/galaxy_tag.html

@@ -0,0 +1,146 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Galaxy - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="my-3">
+                    <a href="{{ url_for('tags_ui.tags_galaxy') }}?galaxy={{ galaxy }}" class="pull-left">
+                        <i class="fa fa-arrow-left fa"></i> {{ galaxy }} Galaxy
+                    </a>
+                </div>
+
+                <div class="card">
+                    <div class="card-header bg-secondary text-white">
+                        <div class="d-flex">
+                            <span class="h3">
+                                {{ tag['value'] }}
+                            </span>
+
+                            <span class="h5 ml-auto">
+                                {% if tag['enabled'] %}
+                                    <span class="badge badge-success">Enabled</span>
+                                {% else %}
+                                    <span class="badge badge-danger">Disabled</span>
+                                {% endif %}
+                            </span>
+                        </div>
+                    </div>
+
+                    <div class="card-body">
+                        <h3><div class="mb-4">
+                            <span class="badge badge-primary">{{ tag['tag'] }}</span>
+                        </div></h3>
+                        <table class="table table-sm table-hover">
+                            <tbody>
+                                {% if 'description' in tag %}
+                                    <tr>
+                                        <td><b>Description</b></td>
+                                    <td>{{ tag['description'] }}</td>
+                                    </tr>
+                                {% endif %}
+                                {% if 'refs' in tag %}
+                                    <tr>
+                                        <td><b>Refs</b></td>
+                                        <td>{{ tag['refs'] }}</td>
+                                    </tr>
+                                {% endif %}
+                            </tbody>
+                        </table>
+
+                        {% if 'meta' in tag %}
+                            <hr>
+                            <b>Meta</b>
+                            <pre class="border">{{ tag['meta'] }}</pre>
+                        {% endif %}
+
+{#                        <div class="mt-4">#}
+{#                            {% if galaxy['enabled'] %}#}
+{#                                <a href="{{ url_for('tags_ui.galaxy_disable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-danger">#}
+{#                                    <i class="fas fa-times"></i> Disable Galaxy#}
+{#                                </a>#}
+{#                            {% else %}#}
+{#                                <a href="{{ url_for('tags_ui.galaxy_enable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-success">#}
+{#                                    <i class="fas fa-check"></i> Enable Galaxy#}
+{#                                </a>#}
+{#                            {% endif %}#}
+{#                        </div>#}
+                    </div>
+                </div>
+
+                <br><br><br><br>
+
+    	    </div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+    $("#navbar-tags").addClass("active");
+    $("#nav_tags_config").removeClass("text-muted");
+
+    table = $('#myTable_').DataTable(
+        {
+            "aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        }
+    );
+
+});
+</script>
+
+<script>
+function submitEnabledTags() {
+    table.destroy();
+	table = $('#myTable_').DataTable(
+		{
+			"iDisplayLength": -1,
+		}
+	);
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+
+</html>

var/www/templates/tags/menu_sidebar.html

@@ -29,26 +29,26 @@
       	      </a>
       	    </li>
             <li class="nav-item">
-      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_screenshot') }}" id="nav_tags_search_screenshot">
       					<i class="fas fa-image"></i>
-      	       	Search Images by Tags
+      	       	Search Screenshots by Tags
       	      </a>
       	    </li>
           </ul>
-      		<h5 class="d-flex text-muted w-100">
+            <h5 class="d-flex text-muted w-100" id="nav_tags_config">
       		  <span>Tags Management </span>
       		</h5>
       	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{ url_for('Tags.taxonomies') }}" id="nav_taxonomies">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_taxonomies') }}" id="nav_taxonomies">
-      	        <i class="fas fa-wrench"></i>
+      	        <i class="fas fa-tags"></i>
-      	        Edit Taxonomies List
+      	        Taxonomies
       	      </a>
       	    </li>
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{ url_for('Tags.galaxies') }}" id="nav_onion_galaxies">
+                <a class="nav-link" href="{{ url_for('tags_ui.tags_galaxies') }}" id="nav_galaxies">
                     <i class="fas fa-rocket"></i>
-      	       	Edit Galaxies List
+                    Galaxies
                 </a>
       	    </li>
           </ul>

var/www/templates/tags/search_obj_by_tags.html

@@ -61,7 +61,7 @@
                     <th>Last seen</th>
                     <th>Nb seen</th>
                     <th>Size</th>
-                  {%elif dict_tagged["object_type"]=="image"%}
+                  {%elif dict_tagged["object_type"]=="screenshot"%}
                     <th style="max-width: 800px;">Image</th>
                   {%elif dict_tagged["object_type"]=="item"%}
                     <th>Date</th>
@@ -106,7 +106,7 @@
   				          </tr>
   				        {% endfor %}
 
-                {%elif dict_tagged["object_type"]=="image"%}
+                {%elif dict_tagged["object_type"]=="screenshot"%}
                   {% for dict_obj in dict_tagged["tagged_obj"] %}
                     <tr>
                       <td class="pb-0">
@@ -141,7 +141,7 @@
                         </div>
                       </td>
                       <td class="pb-0">
-                        <button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#modal_show_min_item" data-url="{{ url_for('showsavedpastes.showsaveditem_min') }}?paste={{ dict_obj['id'] }}" data-path="{{ dict_obj['id'] }}">
+                        <button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#modal_show_min_item" data-url="{{ url_for('objects_item.item_preview') }}?id={{ dict_obj['id'] }}" data-path="{{ dict_obj['id'] }}">
   												<span class="fas fa-search-plus"></span>
   											</button>
 

var/www/templates/tags/taxonomies.html

@@ -0,0 +1,123 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Taxonomies - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <h3 class="mt-2 text-secondary">
+					<i class="fas fa-tags"></i> Taxonomies:
+				</h3>
+
+                <table class="table table-bordered table-striped table-hover" id="myTable_">
+                    <thead class="thead-dark">
+                        <tr>
+                            <th>Namespace</th>
+                            <th>Description</th>
+                            <th>Version</th>
+                            <th>Enabled</th>
+                            <th>Active Tags</th>
+                            <th></th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for taxonomy in taxonomies %}
+                            <tr>
+                                <td>{{ taxonomies[taxonomy]['name'] }}</td>
+                                <td>{{ taxonomies[taxonomy]['description'] }}</td>
+                                <td>{{ taxonomies[taxonomy]['version'] }}</td>
+                                <td>
+                                    {% if taxonomies[taxonomy]['enabled'] %}
+                                        <div style="color:Green;">
+                                            <i class="fas fa-check-circle fa-2x"></i>
+										</div>
+                                    {% else %}
+                                        <div style="color:Red;">
+											<i class="fas fa-times-circle fa-2x"></i>
+										</div>
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    <h5>
+                                        <span class="badge badge-dark">
+                                            <b>{{ taxonomies[taxonomy]['nb_active_tags'] }}</b>/{{ taxonomies[taxonomy]['nb_tags'] }}
+                                        </span>
+                                    </h5>
+                                </td>
+                                <td>
+                                    <a class="btn btn-outline-secondary" href="{{ url_for('tags_ui.tags_taxonomy') }}?taxonomy={{ taxonomy }}">
+                                        <i class="fas fa-tag"></i>
+                                    </a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+
+                </table>
+
+                <br>
+                <br>
+                <br>
+                <br>
+
+    	    </div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+
+$(document).ready(function(){
+  $("#navbar-tags").addClass("active");
+  $("#nav_tags_config").removeClass("text-muted");
+  $("#nav_taxonomies").addClass("active");
+
+
+  $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] });
+
+});
+</script>
+
+<script>
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</html>

var/www/templates/tags/taxonomy.html

@@ -0,0 +1,196 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Taxonomy - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="my-3">
+                    <a href="{{ url_for('tags_ui.tags_taxonomies') }}" class="pull-left">
+                        <i class="fa fa-arrow-left fa"></i> List Taxonomies
+                    </a>
+                </div>
+
+                <div class="card">
+                    <div class="card-header bg-secondary text-white">
+                        <div class="d-flex">
+                            <span class="h3">{{ taxonomy['name'] }}</span>
+
+                            <span class="h5 ml-auto">
+                                {% if taxonomy['enabled'] %}
+                                    <span class="badge badge-success">Enabled</span>
+                                {% else %}
+                                    <span class="badge badge-danger">Disabled</span>
+                                {% endif %}
+                            </span>
+                        </div>
+                    </div>
+
+                    <div class="card-body">
+                        {% if taxonomy['expended'] %}
+                            <div>{{ taxonomy['expended'] }}</div>
+                        {% endif %}
+                        {{ taxonomy['description'] }}
+                        <div>Version: {{ taxonomy['version'] }}</div>
+                        {% if taxonomy['predicates'] %}
+                            <br>
+                            Predicates:
+                            <table class="table table-sm table-hover">
+                                <tbody>
+                                {% for predicate in taxonomy['predicates'] %}
+                                    <tr>
+                                        <td><b>{{ predicate }}</b></td>
+                                        <td>{{ taxonomy['predicates'][predicate]['expanded'] }}</td>
+                                        <td>{{ taxonomy['predicates'][predicate]['description'] }}</td>
+                                    </tr>
+                                {% endfor %}
+                                </tbody>
+                            </table>
+                        {% endif %}
+                        {% if taxonomy['refs'] %}
+                            <div>Refs: {{ taxonomy['refs'] }}</div>
+                            <br><br>
+                        {% endif %}
+
+                        <div class="mt-4">
+                            {% if taxonomy['enabled'] %}
+                                <a href="{{ url_for('tags_ui.taxonomy_disable') }}?taxonomy={{ taxonomy['name'] }}" class="btn btn-danger">
+                                    <i class="fas fa-times"></i> Disable Taxonomy
+                                </a>
+                            {% else %}
+                                <a href="{{ url_for('tags_ui.taxonomy_enable') }}?taxonomy={{ taxonomy['name'] }}" class="btn btn-success">
+                                    <i class="fas fa-check"></i> Enable Taxonomy
+                                </a>
+                            {% endif %}
+                        </div>
+                    </div>
+                </div>
+
+                <form action="{{ url_for('tags_ui.taxonomy_enable_tags') }}" id="checkboxForm">
+				    <input type="hidden" value="{{ taxonomy['name'] }}" name="taxonomy">
+
+                    <table class="table table-bordered table-hover my-4">
+                        <thead class="bg-dark text-white">
+                            <tr>
+                                <th></th>
+                                <th>Tag</th>
+                                <th>Description</th>
+                                <th class="text-center">Enabled</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for tag in taxonomy['tags'] %}
+                                <tr>
+                                    <td class="text-center">
+                                        <div class="custom-control custom-switch">
+                                            <input class="custom-control-input" type="checkbox" name="tags" id="{{ tag['tag'] }}" value="{{ tag['tag'] }}" {% if tag['enabled'] %}checked{% endif %}>
+                                            <label class="custom-control-label" for="{{ tag['tag'] }}"></label>
+                                        </div>
+                                    </td>
+                                    <td><span class="badge badge-info">{{ tag['tag'] }}</span></td>
+                                    <td>
+                                        {% if 'expanded' in tag %}
+                                            {{ tag['expanded'] }}
+                                        {% endif %}
+                                    </td>
+                                    <td>
+                                        <div class="text-center">
+                                            {% if tag['enabled'] %}
+                                                <div style="color:Green;">
+                                                    <i class="fas fa-check-circle"></i>
+                                                </div>
+                                            {% else %}
+                                                <div style="color:Red;">
+                                                    <i class="fas fa-times-circle"></i>
+                                                </div>
+                                            {% endif %}
+                                        </div>
+                                    </td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+
+                    <div class="input-group-btn">
+                        <button class="btn btn-primary btn-lg" onclick="submitEnabledTags()">
+                            <i class="fas fa-check-square"></i> Update Tags
+                        </button>
+                    </div>
+
+                </form>
+
+                <br><br><br><br>
+
+    	    </div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+    $("#navbar-tags").addClass("active");
+    $("#nav_tags_config").removeClass("text-muted");
+
+    table = $('#myTable_').DataTable(
+        {
+            "aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        }
+    );
+
+});
+</script>
+
+<script>
+function submitEnabledTags() {
+    table.destroy();
+	table = $('#myTable_').DataTable(
+		{
+			"iDisplayLength": -1,
+		}
+	);
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+
+</html>