chg: [v4.1] add Investigation with MISP Export + v4.1 update

bin/export/MispExport.py

@@ -22,6 +22,10 @@ import Correlate_object
 import AILObjects
 import Export
 
+
+from Investigations import Investigation
+import Tag
+
 # # TODO: # FIXME: REFRACTOR ME => use UI/Global config
 sys.path.append('../../configs/keys')
 try:
@@ -390,6 +394,7 @@ def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, a
     # # TODO: handle multiple MISP instance
     misp = PyMISP(misp_url, misp_key, misp_verifycert)
     #print(event.to_json())
+
     misp_event = misp.add_event(event)
      #print(misp_event)
     # # TODO: handle error
@@ -414,9 +419,50 @@ def extract_event_metadata(event):
 # LVL 1 => DETAILED   Also add correlated_items correlation
 ######
 
-if __name__ == '__main__':
+# # TODO: # create object relationships
+def create_investigation_event(investigation_uuid):
+    investigation = Investigation(investigation_uuid)
 
-    l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
-    create_list_of_objs_to_export(l_obj)
+    event = MISPEvent()
+    event.info = investigation.get_info()
+    event.uuid = investigation.get_uuid()
+    event.date = investigation.get_date()
+    event.analysis = investigation.get_analysis()
+    event.threat_level_id = investigation.get_threat_level()
+
+    taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(investigation.get_tags())
+    event.Tag = taxonomies_tags
+    event.Galaxy = galaxies_tags
+    #event.add_galaxy(galaxies_tags)
+
+    investigation_objs = investigation.get_objects()
+    for obj in investigation_objs:
+        # if subtype -> obj_id = 'subtype:type'
+        if obj['subtype']:
+            obj_id = f"{obj['subtype']}:{obj['id']}"
+        else:
+            obj_id = obj['id']
+        misp_obj = create_misp_obj(obj['type'], obj_id)
+        if misp_obj:
+            event.add_object(misp_obj)
+
+    # if publish:
+    #     event.publish()
+
+    # res = event.to_json()
+    # print(event.to_json())
+
+    misp = PyMISP(misp_url, misp_key, misp_verifycert)
+    misp_event = misp.add_event(event)
+    # print(misp_event)
+
+    # # TODO: handle error
+    event_metadata = extract_event_metadata(misp_event)
+    return event_metadata
+
+# if __name__ == '__main__':
+
+    # l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
+    # create_list_of_objs_to_export(l_obj)
 
     #print(event.to_json())

bin/lib/Config_DB.py

@@ -24,44 +24,116 @@ config_loader = None
 ## data retention
 #########################
 
-default_config = {
+
+ail_config = {
     "crawler": {
-        "enable_har_by_default": False,
-        "enable_screenshot_by_default": True,
-        "default_depth_limit": 1,
-        "default_closespider_pagecount": 50,
-        "default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0",
-        "default_timeout": 30
+        "enable_har_by_default": {
+            "default": False,
+            "type": bool,
+            "info": "Enable HAR by default"
+        },
+        "enable_screenshot_by_default": {
+            "default": True,
+            "type": bool,
+            "info": "Enable screenshot by default"
+        },
+        "depth_limit": {
+            "default": 1,
+            "type": int,
+            "info": "Maximum number of url depth"
+        },
+        "closespider_pagecount": {
+            "default": 50,
+            "type": int,
+            "info": "Maximum number of pages"
+        },
+        "user_agent": {
+            "default": 50,
+            "type": str,
+            "info": "User agent used by default"
+        },
+        "timeout": {
+            "default": 30,
+            "type": int,
+            "info": "Crawler connection timeout"
+        },
+    },
+    "misp": {
+        "url": {
+            "default": "https://localhost:8443/",
+            "type": str,
+            "info": "Crawler connection timeout"
+        },
+        "key": {
+            "default": "",
+            "type": str,
+            "info": "Crawler connection timeout"
+        },
+        "verifycert": {
+            "default": True,
+            "type": bool,
+            "info": "Crawler connection timeout"
+        },
     }
 }
 
+# The MISP auth key can be found on the MISP web interface under the automation section
+
+def get_config_value(section, field, value):
+    return r_serv_db.hget(f'ail:config:global:{section}', field, value)
+
+def get_config_default_value(section, field, value):
+    return ail_config[section][field]['default']
+
+def get_config_type(section, field, value):
+    return ail_config[section][field]['type']
+
+def get_config_info(section, field, value):
+    return ail_config[section][field]['info']
+
+def save_config(section, field, value):
+    if section in ail_config:
+        if is_valid_type(value, section, field, value_type=value_type):
+            # if value_type in ['list', 'set', 'dict']:
+            #     pass
+            # else:
+            r_serv_db.hset(f'ail:config:global:{section}', field, value)
+
+
+config_documentation = {
+
+}
+
+default_config = {
+
+}
+
 def get_default_config():
     return default_config
 
 def get_default_config_value(section, field):
     return default_config[section][field]
 
-config_type = {
-    # crawler config
-    "crawler": {
-        "enable_har_by_default": bool,
-        "enable_screenshot_by_default": bool,
-        "default_depth_limit": int,
-        "default_closespider_pagecount": int,
-        "default_user_agent": str,
-        "default_timeout": int
-    }
-}
 
-def get_config_type(section, field):
-    return config_type[section][field]
+
+
+#### DEFAULT CONFIG ####
+
+#### CONFIG TYPE ####
+# CONFIG DOC
+config_type = {
+
+}
 
 # # TODO: add set, dict, list and select_(multiple_)value
 def is_valid_type(obj, section, field, value_type=None):
     res = isinstance(obj, get_config_type(section, field))
     return res
 
+# # TODO: ###########################################################
 def reset_default_config():
+    for section in config_type:
+
     pass
 
 def set_default_config(section, field):
@@ -92,45 +164,12 @@ def get_config_dict_by_section(section):
         config_dict[field] = get_config(section, field)
     return config_dict
 
-def save_config(section, field, value, value_type=None): ###########################################
-    if section in default_config:
-        if is_valid_type(value, section, field, value_type=value_type):
-            if value_type in ['list', 'set', 'dict']:
-                pass
-            else:
-                r_serv_db.hset(f'config:global:{section}', field, value)
-                # used by check_integrity
-                r_serv_db.sadd('config:all_global_section', field, value)
 
 # check config value + type
 def check_integrity():
     pass
 
 
-config_documentation = {
-    "crawler": {
-        "enable_har_by_default": 'Enable HAR by default',
-        "enable_screenshot_by_default": 'Enable screenshot by default',
-        "default_depth_limit": 'Maximum number of url depth',
-        "default_closespider_pagecount": 'Maximum number of pages',
-        "default_user_agent": "User agent used by default",
-        "default_timeout": "Crawler connection timeout"
-    }
-}
-
-def get_config_documentation(section, field):
-    return config_documentation[section][field]
-
-# def conf_view():
-#     class F(MyBaseForm):
-#         pass
-#
-#     F.username = TextField('username')
-#     for name in iterate_some_model_dynamically():
-#         setattr(F, name, TextField(name.title()))
-#
-#     form = F(request.POST, ...)
-
 def get_field_full_config(section, field):
     dict_config = {}
     dict_config['value'] = get_config(section, field)

bin/lib/Investigations.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+##################################################################
+##################################################################
+
+# TODO:  /!\ MISP ORG UUID
+
+##################################################################
+##################################################################
+
+import os
+import sys
+import datetime
+import redis
+import time
+import uuid
+
+from abc import ABC
+from enum import Enum
+from flask import escape
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+from exceptions import UpdateInvestigationError
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Tag
+
+config_loader = ConfigLoader.ConfigLoader()
+r_tracking = config_loader.get_redis_conn("DB_Tracking")
+config_loader = None
+
+
+#### UUID ####
+def is_valid_uuid_v4(UUID):
+    if not UUID:
+        return False
+    UUID = UUID.replace('-', '')
+    try:
+        uuid_test = uuid.UUID(hex=UUID, version=4)
+        return uuid_test.hex == UUID
+    except:
+        return False
+
+def sanityze_uuid(UUID):
+    sanityzed_uuid = uuid.UUID(hex=UUID, version=4)
+    return str(sanityzed_uuid).replace('-', '')
+
+def exists_obj_type(obj_type):
+    return obj_type in ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
+
+def generate_uuid():
+    return str(uuid.uuid4()).replace('-', '')
+
+##-- UUID --##
+
+# status
+# created
+# last change
+# tags
+# comment/info
+# level
+
+## threat_level:
+# 1 = high
+# 2 = medium
+# 3 = low
+# 4 = undefined
+
+## analysis:
+# 0 = Initial
+# 1 = Ongoing
+# 2 = Complete
+
+# # TODO: Save correlation between investigations ?
+
+class ThreatLevel(Enum):
+    high = 1
+    medium = 2
+    low = 3
+    undefined = 4
+
+class Analysis(Enum):
+    initial = 0
+    ongoing = 1
+    completed = 2
+
+class Investigation(object):
+    """Investigation."""
+
+    def __init__(self, investigation_uuid):
+        self.uuid = investigation_uuid
+
+    def get_uuid(self):
+        return self.uuid
+
+    # # TODO: Replace by title ??????
+    def get_name(self):
+        return r_tracking.hget(f'investigations:data:{self.uuid}', 'name')
+
+    def get_threat_level(self):
+        try:
+            return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'threat_level'))
+        except:
+            return 1
+
+    def get_threat_level_str(self):
+        return ThreatLevel(self.get_threat_level()).name
+
+    def get_analysis(self):
+        try:
+            return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'analysis'))
+        except:
+            return 0
+
+    def get_analysis_str(self):
+        return Analysis(self.get_analysis()).name
+
+    def get_tags(self):
+        return r_tracking.smembers(f'investigations:tags:{self.uuid}')
+
+    # save all editor ??????
+    def get_creator_user(self):
+        return r_tracking.hget(f'investigations:data:{self.uuid}', 'creator_user')
+
+    def get_info(self):
+        return r_tracking.hget(f'investigations:data:{self.uuid}', 'info')
+
+    def get_date(self):
+        return r_tracking.hget(f'investigations:data:{self.uuid}', 'date')
+
+    def get_timestamp(self, r_str=False):
+        timestamp = r_tracking.hget(f'investigations:data:{self.uuid}', 'timestamp')
+        if r_str and timestamp:
+            timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
+        return timestamp
+
+    def get_last_change(self, r_str=False):
+        last_change = r_tracking.hget(f'investigations:data:{self.uuid}', 'last_change')
+        if r_str and last_change:
+            last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S')
+        return last_change
+
+    # # TODO: DATE FORMAT
+    def get_metadata(self, r_str=False):
+        if r_str:
+            analysis = self.get_analysis_str()
+            threat_level = self.get_threat_level_str()
+        else:
+            analysis = self.get_analysis()
+            threat_level = self.get_threat_level()
+        return {'uuid': self.uuid,
+                'name': self.get_name(),
+                'threat_level': threat_level,
+                'analysis': analysis,
+                'tags': self.get_tags(),
+                'user_creator': self.get_creator_user(),
+                'date': self.get_date(),
+                'timestamp': self.get_timestamp(r_str=r_str),
+                'last_change': self.get_last_change(r_str=r_str),
+                'info': self.get_info(),
+                'nb_objects': self.get_nb_objects()}
+
+    def set_name(self, name):
+        r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name)
+
+    def set_info(self, info):
+        r_tracking.hset(f'investigations:data:{self.uuid}', 'info', info)
+
+    def set_date(self, date):
+        r_tracking.hset(f'investigations:data:{self.uuid}', 'date', date)
+
+    def set_last_change(self, last_change):
+        r_tracking.hset(f'investigations:data:{self.uuid}', 'last_change', last_change)
+
+    def set_threat_level(self, threat_level):
+        try:
+            threat_level = int(threat_level)
+        except:
+            raise UpdateInvestigationError('threat_level Not an integer')
+        if threat_level >= 1 and threat_level <= 4:
+            r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
+        else:
+            raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
+
+    def set_analysis(self, analysis):
+        try:
+            analysis = int(analysis)
+        except:
+            raise UpdateInvestigationError('analysis Not an integer')
+        if analysis >= 0 and analysis <= 2:
+            r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
+        else:
+            raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
+
+    def set_tags(self, tags):
+        # delete previous tags
+        r_tracking.delete(f'investigations:tags:{self.uuid}')
+        for tag in tags:
+            r_tracking.sadd(f'investigations:tags:{self.uuid}', tag)
+
+    def get_nb_objects(self):
+        return r_tracking.scard(f'investigations:objs:{self.uuid}')
+
+    def _get_objects(self):
+        return r_tracking.smembers(f'investigations:objs:{self.uuid}')
+
+    # # TODO: return Python object ???? ################################
+    # TODO: PAGINATE
+    def get_objects(self):
+        # obj_dict = {}
+        # for str_obj in self._get_objects():
+        #     obj_type, subtype, id = str_obj.split(':', 2)
+        #     if not obj_dict[obj_type]:
+        #         obj_dict[obj_type] = []
+        #     obj_dict[obj_type].append({'subtype': subtype, 'id': id})
+        objs = []
+        for str_obj in self._get_objects():
+            obj_type, subtype, obj_id = str_obj.split(':', 2)
+            dict_obj = {'type': obj_type, 'subtype': subtype, 'id': obj_id}
+            objs.append(dict_obj)
+        return objs
+
+    # # TODO:  def register_object(self, Object): in OBJECT CLASS
+
+    def register_object(self, obj_id, obj_type, subtype):
+        r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
+        r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
+        timestamp = int(time.time())
+        self.set_last_change(timestamp)
+
+
+    def unregister_object(self, obj_id, obj_type, subtype):
+        r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
+        r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
+        timestamp = int(time.time())
+        self.set_last_change(timestamp)
+
+    def delete(self):
+        for str_obj in self._get_objects():
+            obj_type, subtype, obj_id = str_obj.split(':', 2)
+            self.unregister_object(obj_id, obj_type, subtype=subtype)
+
+        r_tracking.srem('investigations:all', self.uuid)
+        # user map
+        r_tracking.srem(f'investigations:user:{self.get_creator_user()}', self.uuid)
+        # metadata
+        r_tracking.delete(f'investigations:data:{self.uuid}')
+        r_tracking.delete(f'investigations:tags:{self.uuid}')
+
+##--  Class  --##
+
+def get_all_investigations():
+    return r_tracking.smembers('investigations:all')
+
+def exists_investigation(investigation_uuid):
+    return r_tracking.sismember('investigations:all', investigation_uuid)
+
+# created by user
+def get_user_all_investigations(user_id):
+    return r_tracking.smembers('investigations:user:{user_id}')
+
+def is_object_investigated(obj_id, obj_type, subtype=''):
+    return r_tracking.exists(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
+
+def get_obj_investigations(obj_id, obj_type, subtype=''):
+    return r_tracking.smembers(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
+
+# # TODO: fix default threat_level analysis
+# # TODO: limit description + name
+# # TODO: sanityze tags
+# # TODO: sanityze date
+def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[]):
+    investigation_uuid = generate_uuid()
+    r_tracking.sadd('investigations:all', investigation_uuid)
+    # user map
+    r_tracking.sadd('investigations:user:{user_id}', investigation_uuid)
+    # metadata
+    r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', user_id)
+
+    # TODO: limit info + name
+    investigation = Investigation(investigation_uuid)
+    investigation.set_info(info)
+    #investigation.set_name(name) ##############################################
+    investigation.set_date(date)
+    investigation.set_threat_level(threat_level)
+    investigation.set_analysis(analysis)
+
+    # # TODO: sanityze tags
+    if tags:
+        investigation.set_tags(tags)
+
+    timestamp = int(time.time())
+    r_tracking.hset(f'investigations:data:{investigation_uuid}', 'timestamp', timestamp)
+    investigation.set_last_change(timestamp)
+
+    return investigation_uuid
+
+def get_all_investigations_meta(r_str=False):
+    investigations_meta = []
+    for investigation_uuid in get_all_investigations():
+        investigation = Investigation(investigation_uuid)
+        investigations_meta.append(investigation.get_metadata(r_str=r_str))
+    return investigations_meta
+
+def get_investigations_selector():
+    l_investigations = []
+    for investigation_uuid in get_all_investigations():
+        investigation = Investigation(investigation_uuid)
+        name = investigation.get_info()
+        l_investigations.append({"id":investigation_uuid, "name": name})
+    return l_investigations
+
+    #{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"}
+
+
+####  API  ####
+
+# # TODO: CHECK Mandatory Fields
+# # TODO: SANITYZE Fields
+# # TODO: Name ?????
+def api_add_investigation(json_dict):
+    user_id = json_dict.get('user_id')
+    name = json_dict.get('name') ##### mandatory ?
+    name = escape(name)
+    threat_level = json_dict.get('threat_level', 4)
+    analysis = json_dict.get('analysis', 0)
+
+    # # TODO: sanityze date
+    date = json_dict.get('date')
+
+    info = json_dict.get('info', '')
+    info = escape(info)
+    info = info[:1000]
+    tags = json_dict.get('tags', [])
+    if not Tag.are_enabled_tags(tags):
+        return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
+
+    try:
+        res = create_investigation(user_id, date, name, threat_level, analysis, info, tags=tags)
+    except UpdateInvestigationError as e:
+        return e.message, 400
+    return res, 200
+
+# # TODO: edit threat level / status
+def api_edit_investigation(json_dict):
+    investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
+    if not is_valid_uuid_v4(investigation_uuid):
+        return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
+    investigation_uuid = sanityze_uuid(investigation_uuid)
+    if not exists_investigation(investigation_uuid):
+        return {"status": "error", "reason": "Investigation not found"}, 404
+    investigation = Investigation(investigation_uuid)
+
+    name = json_dict.get('name') ##### mandatory ?
+    name = escape(name)
+    threat_level = json_dict.get('threat_level', 4)
+    try:
+        investigation.set_threat_level(threat_level)
+    except UpdateInvestigationError:
+        return {"status": "error", "reason": "Invalid Investigation threat_level"}, 400
+
+    analysis = json_dict.get('analysis', 0)
+    try:
+        investigation.set_analysis(analysis)
+    except UpdateInvestigationError:
+        return {"status": "error", "reason": "Invalid Investigation analysis"}, 400
+
+    info = json_dict.get('info', '')
+    info = escape(info)
+    info = info[:1000]
+    tags = json_dict.get('tags', [])
+    if not Tag.are_enabled_tags(tags):
+        return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
+
+    investigation.set_info(info)
+    investigation.set_name(name)
+    investigation.set_tags(tags)
+
+    timestamp = int(time.time())
+    investigation.set_last_change(timestamp)
+
+    return investigation_uuid, 200
+
+def api_delete_investigation(json_dict):
+    investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
+    if not is_valid_uuid_v4(investigation_uuid):
+        return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
+    investigation_uuid = sanityze_uuid(investigation_uuid)
+    if not exists_investigation(investigation_uuid):
+        return {"status": "error", "reason": "Investigation not found"}, 404
+    investigation = Investigation(investigation_uuid)
+    res = investigation.delete()
+    return res, 200
+
+def api_register_object(json_dict):
+    investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
+    if not is_valid_uuid_v4(investigation_uuid):
+        return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
+    investigation_uuid = sanityze_uuid(investigation_uuid)
+    if not exists_investigation(investigation_uuid):
+        return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
+    investigation = Investigation(investigation_uuid)
+
+    obj_type = json_dict.get('type', '').replace(' ', '')
+    if not exists_obj_type(obj_type):
+        return {"status": "error", "reason": f"Invalid Object Type: {obj_type}"}, 400
+
+    subtype = json_dict.get('subtype', '')
+    if subtype == 'None':
+        subtype = ''
+    obj_id = json_dict.get('id', '').replace(' ', '')
+    res = investigation.register_object(obj_id, obj_type, subtype)
+    return res, 200
+
+def api_unregister_object(json_dict):
+    investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
+    if not is_valid_uuid_v4(investigation_uuid):
+        return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
+    investigation_uuid = sanityze_uuid(investigation_uuid)
+    if not exists_investigation(investigation_uuid):
+        return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
+    investigation = Investigation(investigation_uuid)
+
+    obj_type = json_dict.get('type', '').replace(' ', '')
+    subtype = json_dict.get('subtype', '')
+    if subtype == 'None':
+        subtype = ''
+    obj_id = json_dict.get('id', '').replace(' ', '')
+    res = investigation.unregister_object(obj_id, obj_type, subtype)
+    return res, 200
+
+##--  API  --##
+
+
+if __name__ == '__main__':
+    # res = create_star_list(user_id, name, description)
+    # print(res)
+
+    # res = r_tracking.dbsize()
+    # print(res)
+
+    investigation_uuid = 'e4e1c8e3b0a349bf81482f2f823efc0f'
+
+    investigation = Investigation(investigation_uuid)
+    investigation.delete()
+
+# # TODO: PAGINATION

bin/lib/ail_objects.py

@@ -1,406 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import sys
-import uuid
-import redis
-
-from abc import ABC
-from flask import url_for
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-import Tag
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
-import ConfigLoader
-
-config_loader = ConfigLoader.ConfigLoader()
-r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
-config_loader = None
-
-class AbstractObject(ABC):
-    """
-    Abstract Object
-    """
-
-    # first seen last/seen ??
-    # # TODO: - tags
-    #         - handle + refactor coorelations
-    #         - creates others objects
-
-    def __init__(self, obj_type, id, subtype=None):
-        """ Abstract for all the AIL object
-
-        :param obj_type: object type (item, ...)
-        :param id: Object ID
-        """
-        self.id = id
-        self.type = obj_type
-        self.subtype = None
-
-    def get_id(self):
-        return self.id
-
-    def get_type(self):
-        return self.type
-
-    def get_subtype(self, r_str=False):
-        if not self.subtype:
-            if r_str:
-                return ''
-        return self.subtype
-
-    def get_default_meta(self):
-        dict_meta = {'id': self.get_id(),
-                     'type': self.get_type()}
-        if self.subtype:
-            dict_meta['subtype'] = self.subtype
-        return dict_meta
-
-    def get_tags(self, r_set=False):
-        tags = Tag.get_obj_tag(self.id)
-        if r_set:
-            tags = set(tags)
-        return tags
-
-    ## ADD TAGS ????
-    #def add_tags(self):
-
-    def _delete(self):
-        # DELETE TAGS
-        Tag.delete_obj_all_tags(self.id, self.type)
-        if self.type == 'item':
-            # delete tracker
-            pass
-
-
-def is_valid_object_type(object_type):
-    if object_type in ['domain', 'item', 'image', 'decoded']:
-        return True
-    else:
-        return False
-
-def get_all_objects():
-    return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
-
-def get_all_correlation_names():
-    '''
-    Return a list of all available correlations
-    '''
-    return ['pgp', 'cryptocurrency', 'decoded', 'screenshot']
-
-def get_all_correlation_objects():
-    '''
-    Return a list of all correllated objects
-    '''
-    return ['domain', 'paste']
-
-def exist_object(object_type, correlation_id, type_id=None):
-    if object_type == 'domain':
-        return Domain.verify_if_domain_exist(correlation_id)
-    elif object_type == 'paste' or object_type == 'item':
-        return Item.exist_item(correlation_id)
-    elif object_type == 'decoded':
-        return Decoded.exist_decoded(correlation_id)
-    elif object_type == 'pgp':
-        return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
-    elif object_type == 'cryptocurrency':
-        return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
-    elif object_type == 'screenshot' or object_type == 'image':
-        return Screenshot.exist_screenshot(correlation_id)
-    else:
-        return False
-
-def get_obj_date(object_type, object_id):
-    if object_type == "item":
-        return int(Item.get_item_date(object_id))
-    else:
-        return None
-
-# request_type => api or ui
-def get_object_metadata(object_type, correlation_id, type_id=None):
-    if object_type == 'domain':
-        return Domain.Domain(correlation_id).get_domain_metadata(tags=True)
-    elif object_type == 'paste' or object_type == 'item':
-        return Item.get_item({"id": correlation_id, "date": True, "date_separator": True, "tags": True})[0]
-    elif object_type == 'decoded':
-        return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True, file_type=True, tag=True)
-    elif object_type == 'pgp':
-        return Pgp.pgp.get_metadata(type_id, correlation_id)
-    elif object_type == 'cryptocurrency':
-        return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
-    elif object_type == 'screenshot' or object_type == 'image':
-        return Screenshot.get_metadata(correlation_id)
-
-def get_object_correlation(object_type, value, correlation_names=None, correlation_objects=None, requested_correl_type=None):
-    if object_type == 'domain':
-        return Domain.get_domain_all_correlation(value, correlation_names=correlation_names)
-    elif object_type == 'paste' or object_type == 'item':
-        return Item.get_item_all_correlation(value, correlation_names=correlation_names)
-    elif object_type == 'decoded':
-        return Decoded.get_decoded_correlated_object(value, correlation_objects=correlation_objects)
-    elif object_type == 'pgp':
-        return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
-    elif object_type == 'cryptocurrency':
-        return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
-    elif object_type == 'screenshot' or object_type == 'image':
-        return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects)
-    return {}
-
-def get_correlation_node_icon(correlation_name, correlation_type=None, value=None):
-    '''
-    Used in UI Graph.
-    Return a font awesome icon for a given correlation_name.
-
-    :param correlation_name: correlation name
-    :param correlation_name: str
-    :param correlation_type: correlation type
-    :type correlation_type: str, optional
-
-    :return: a dictionnary {font awesome class, icon_code}
-    :rtype: dict
-    '''
-    icon_class = 'fas'
-    icon_text = ''
-    node_color = "#332288"
-    node_radius = 6
-    if correlation_name == "pgp":
-        node_color = '#44AA99'
-        if correlation_type == 'key':
-            icon_text = '\uf084'
-        elif correlation_type == 'name':
-            icon_text = '\uf507'
-        elif correlation_type == 'mail':
-            icon_text = '\uf1fa'
-        else:
-            icon_text = 'times'
-
-    elif correlation_name == 'cryptocurrency':
-        node_color = '#DDCC77'
-        if correlation_type == 'bitcoin':
-            icon_class = 'fab'
-            icon_text = '\uf15a'
-        elif correlation_type == 'monero':
-            icon_class = 'fab'
-            icon_text = '\uf3d0'
-        elif correlation_type == 'ethereum':
-            icon_class = 'fab'
-            icon_text = '\uf42e'
-        else:
-            icon_text = '\uf51e'
-
-    elif correlation_name == 'decoded':
-        node_color = '#88CCEE'
-        correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
-        if correlation_type == 'application':
-            icon_text = '\uf15b'
-        elif correlation_type == 'audio':
-            icon_text = '\uf1c7'
-        elif correlation_type == 'image':
-            icon_text = '\uf1c5'
-        elif correlation_type == 'text':
-            icon_text = '\uf15c'
-        else:
-            icon_text = '\uf249'
-
-    elif correlation_name == 'screenshot' or correlation_name == 'image':
-        node_color = '#E1F5DF'
-        icon_text = '\uf03e'
-
-    elif correlation_name == 'domain':
-        node_radius = 5
-        node_color = '#3DA760'
-        if Domain.get_domain_type(value) == 'onion':
-            icon_text = '\uf06e'
-        else:
-            icon_class = 'fab'
-            icon_text = '\uf13b'
-
-    elif correlation_name == 'paste':
-        node_radius = 5
-        if Item.is_crawled(value):
-            node_color = 'red'
-        else:
-            node_color = '#332288'
-
-    return {"icon_class": icon_class, "icon_text": icon_text, "node_color": node_color, "node_radius": node_radius}
-
-def get_item_url(correlation_name, value, correlation_type=None):
-    '''
-    Warning: use only in flask
-    '''
-    url = '#'
-    if correlation_name == "pgp":
-        endpoint = 'correlation.show_correlation'
-        url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value)
-    elif correlation_name == 'cryptocurrency':
-        endpoint = 'correlation.show_correlation'
-        url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
-    elif correlation_name == 'decoded':
-        endpoint = 'correlation.show_correlation'
-        url = url_for(endpoint, object_type="decoded", correlation_id=value)
-    elif correlation_name == 'screenshot' or correlation_name == 'image':              ### # TODO:  rename me
-        endpoint = 'correlation.show_correlation'
-        url = url_for(endpoint, object_type="screenshot", correlation_id=value)
-    elif correlation_name == 'domain':
-        endpoint = 'crawler_splash.showDomain'
-        url = url_for(endpoint, domain=value)
-    elif correlation_name == 'item':
-        endpoint = 'showsavedpastes.showsavedpaste'
-        url = url_for(endpoint, paste=value)
-    elif correlation_name == 'paste':                   ### # TODO:  remove me
-        endpoint = 'showsavedpastes.showsavedpaste'
-        url = url_for(endpoint, paste=value)
-    return url
-
-def get_obj_tag_table_keys(object_type):
-    '''
-    Warning: use only in flask (dynamic templates)
-    '''
-    if object_type=="domain":
-        return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
-
-
-def create_graph_links(links_set):
-    graph_links_list = []
-    for link in links_set:
-        graph_links_list.append({"source": link[0], "target": link[1]})
-    return graph_links_list
-
-def create_graph_nodes(nodes_set, root_node_id):
-    graph_nodes_list = []
-    for node_id in nodes_set:
-        correlation_name, correlation_type, value = node_id.split(';', 3)
-        dict_node = {"id": node_id}
-        dict_node['style'] = get_correlation_node_icon(correlation_name, correlation_type, value)
-        dict_node['text'] = value
-        if node_id == root_node_id:
-            dict_node["style"]["node_color"] = 'orange'
-            dict_node["style"]["node_radius"] = 7
-        dict_node['url'] = get_item_url(correlation_name, value, correlation_type)
-        graph_nodes_list.append(dict_node)
-    return graph_nodes_list
-
-def create_node_id(correlation_name, value, correlation_type=''):
-    if correlation_type is None:
-        correlation_type = ''
-    return '{};{};{}'.format(correlation_name, correlation_type, value)
-
-
-
-# # TODO: filter by correlation type => bitcoin, mail, ...
-def get_graph_node_object_correlation(object_type, root_value, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None):
-    links = set()
-    nodes = set()
-
-    root_node_id = create_node_id(object_type, root_value, requested_correl_type)
-    nodes.add(root_node_id)
-
-    root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
-    for correl in root_correlation:
-        if correl in ('pgp', 'cryptocurrency'):
-            for correl_type in root_correlation[correl]:
-                for correl_val in root_correlation[correl][correl_type]:
-
-                    # add correlation
-                    correl_node_id = create_node_id(correl, correl_val, correl_type)
-
-                    if mode=="union":
-                        if len(nodes) > max_nodes:
-                            break
-                        nodes.add(correl_node_id)
-                        links.add((root_node_id, correl_node_id))
-
-                    # get second correlation
-                    res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type)
-                    if res:
-                        for corr_obj in res:
-                            for correl_key_val in res[corr_obj]:
-                                #filter root value
-                                if correl_key_val == root_value:
-                                    continue
-
-                                if len(nodes) > max_nodes:
-                                    break
-                                new_corel_1 = create_node_id(corr_obj, correl_key_val)
-                                new_corel_2 = create_node_id(correl, correl_val, correl_type)
-                                nodes.add(new_corel_1)
-                                nodes.add(new_corel_2)
-                                links.add((new_corel_1, new_corel_2))
-
-                                if mode=="inter":
-                                    nodes.add(correl_node_id)
-                                    links.add((root_node_id, correl_node_id))
-        if correl in ('decoded', 'screenshot', 'domain', 'paste'):
-            for correl_val in root_correlation[correl]:
-
-                correl_node_id = create_node_id(correl, correl_val)
-                if mode=="union":
-                    if len(nodes) > max_nodes:
-                        break
-                    nodes.add(correl_node_id)
-                    links.add((root_node_id, correl_node_id))
-
-                res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects)
-                if res:
-                    for corr_obj in res:
-                        if corr_obj in ('decoded', 'domain', 'paste', 'screenshot'):
-                            for correl_key_val in res[corr_obj]:
-                                #filter root value
-                                if correl_key_val == root_value:
-                                    continue
-
-                                if len(nodes) > max_nodes:
-                                    break
-                                new_corel_1 = create_node_id(corr_obj, correl_key_val)
-                                new_corel_2 = create_node_id(correl, correl_val)
-                                nodes.add(new_corel_1)
-                                nodes.add(new_corel_2)
-                                links.add((new_corel_1, new_corel_2))
-
-                                if mode=="inter":
-                                    nodes.add(correl_node_id)
-                                    links.add((root_node_id, correl_node_id))
-
-                        if corr_obj in ('pgp', 'cryptocurrency'):
-                            for correl_key_type in res[corr_obj]:
-                                for correl_key_val in res[corr_obj][correl_key_type]:
-                                    #filter root value
-                                    if correl_key_val == root_value:
-                                        continue
-
-                                    if len(nodes) > max_nodes:
-                                        break
-                                    new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type)
-                                    new_corel_2 = create_node_id(correl, correl_val)
-                                    nodes.add(new_corel_1)
-                                    nodes.add(new_corel_2)
-                                    links.add((new_corel_1, new_corel_2))
-
-                                    if mode=="inter":
-                                        nodes.add(correl_node_id)
-                                        links.add((root_node_id, correl_node_id))
-
-
-    return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)}
-
-
-def get_obj_global_id(obj_type, obj_id, obj_sub_type=None):
-    if obj_sub_type:
-        return '{}:{}:{}'.format(obj_type, obj_sub_type, obj_id)
-    else:
-        # # TODO: remove me
-        if obj_type=='paste':
-            obj_type='item'
-        # # TODO: remove me
-        if obj_type=='screenshot':
-            obj_type='image'
-
-        return '{}:{}'.format(obj_type, obj_id)
-
-######## API EXPOSED ########
-def sanitize_object_type(object_type):
-    if not is_valid_object_type(object_type):
-        return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
-########  ########

bin/lib/ail_users.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import uuid
+import redis
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
+
+class User(object):
+    """AIL User."""
+
+    def __init__(self, id):
+        self.id = id
+        if self.id == '__anonymous__':
+            self.role = 'anonymous'
+        else:
+            self.role = None
+
+    def get_role(self):
+        pass
+
+    

bin/lib/exceptions.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+class AIL_ERROR(Exception):
+    """docstring for AIL_ERROR."""
+
+    def __init__(self, message):
+        super(AIL_ERROR, self).__init__(message)
+        self.message = message
+
+class UpdateInvestigationError(AIL_ERROR):
+    pass
+
+class NewTagError(AIL_ERROR):
+    pass

bin/lib/objects/CryptoCurrencies.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class CryptoCurrency(AbstractObject):
+    """
+    AIL CryptoCurrency Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(CryptoCurrency, self).__init__('cryptocurrency', id, subtype=subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        if self.subtype == 'bitcoin':
+            style = 'fab'
+            icon = '\uf15a'
+        elif self.subtype == 'monero':
+            style = 'fab'
+            icon = '\uf3d0'
+        elif self.subtype == 'ethereum':
+            style = 'fab'
+            icon = '\uf42e'
+        else:
+            style = 'fas'
+            icon = '\uf51e'
+        return {'style': style, 'icon': icon, 'color': '#DDCC77', 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/Decodeds.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+HASH_DIR = config_loader.get_config_str('Directories', 'hash')
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+# # TODO: COMPLETE CLASS
+
+class Decoded(AbstractObject):
+    """
+    AIL Decoded Object. (strings)
+    """
+
+    def __init__(self, id):
+        super(Decoded, self).__init__('decoded', id)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    def get_sha1(self):
+        return self.id.split('/')[0]
+
+    def get_file_type(self):
+        return r_serv_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type')
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', object_type="decoded", correlation_id=value)
+        else:
+            url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        file_type = self.get_file_type()
+        if file_type == 'application':
+            icon = '\uf15b'
+        elif file_type == 'audio':
+            icon = '\uf1c7'
+        elif file_type == 'image':
+            icon = '\uf1c5'
+        elif file_type == 'text':
+            icon = '\uf15c'
+        else:
+            icon = '\uf249'
+        return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/Domains.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Domain(AbstractObject):
+    """
+    AIL Decoded Object. (strings)
+    """
+
+    def __init__(self, id):
+        super(Domain, self).__init__('domain', id)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    def get_domain_type(self):
+        if str(self.id).endswith('.onion'):
+            return 'onion'
+        else:
+            return 'regular'
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('crawler_splash.showDomain', domain=self.id)
+        else:
+            url = f'{baseurl}/crawlers/showDomain?domain={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        color = '#3DA760'
+        if self.get_domain_type() == 'onion':
+            style = 'fas'
+            icon = '\uf06e'
+        else:
+            style = 'fab'
+            icon = '\uf13b'
+        return {'style': style, 'icon': icon, 'color':color, 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/Items.py

@@ -0,0 +1,690 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import base64
+import os
+import re
+import sys
+import redis
+import cld3
+import html2text
+
+from io import BytesIO
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Tag
+import Cryptocurrency
+import Pgp
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import item_basic
+import domain_basic
+import ConfigLoader
+import Correlate_object
+import Decoded
+import Screenshot
+import Username
+
+from abstract_object import AbstractObject
+from item_basic import *
+
+config_loader = ConfigLoader.ConfigLoader()
+# get and sanityze PASTE DIRECTORY
+# # TODO: rename PASTES_FOLDER
+PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
+PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
+
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+screenshot_directory = config_loader.get_files_directory('screenshot')
+har_directory = config_loader.get_files_directory('har')
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Item(AbstractObject):
+    """
+    AIL Item Object. (strings)
+    """
+
+    def __init__(self, id):
+        super(Item, self).__init__('item', id)
+
+    def get_date(self, separator=False):
+        """
+        Returns Item date
+        """
+        return item_basic.get_item_date(self.id, add_separator=separator)
+
+    def get_source(self):
+        """
+        Returns Item source/feeder name
+        """
+        #return self.id.split('/')[-5]
+        l_source = self.id.split('/')[:-4]
+        return os.path.join(*l_source)
+
+    def get_basename(self):
+        return os.path.basename(self.id)
+
+    def get_filename(self):
+        # Creating the full filepath
+        filename = os.path.join(PASTES_FOLDER, self.id)
+        filename = os.path.realpath(filename)
+
+        # incorrect filename
+        if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
+            return None
+        else:
+            return filename
+
+    def get_content(self):
+        """
+        Returns Item content
+        """
+        return item_basic.get_item_content(self.id)
+
+    def get_gzip_content(self, b64=False):
+        with open(self.get_filename(), 'rb') as f:
+            content = f.read()
+        if b64:
+            content = base64.b64encode(content)
+        return content.decode()
+
+    def get_ail_2_ail_payload(self):
+        payload = {'raw': self.get_gzip_content(b64=True),
+                    'compress': 'gzip'}
+        return payload
+
+    # # TODO:
+    def create(self):
+        pass
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    # TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ...
+    def delete(self):
+        try:
+            os.remove(self.get_filename())
+            return True
+        except FileNotFoundError:
+            return False
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_item.showItem', id=value)
+        else:
+            url = f'{baseurl}/object/item?id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        if is_crawled(self.id):
+            color = 'red'
+        else:
+            color = '#332288'
+        return {'style': '', 'icon': '', 'color': color, 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+################################################################################
+################################################################################
+################################################################################
+
+def exist_item(item_id):
+    return item_basic.exist_item(item_id)
+
+def get_basename(item_id):
+    return os.path.basename(item_id)
+
+def get_item_id(full_path):
+    return full_path.replace(PASTES_FOLDER, '', 1)
+
+def get_item_filepath(item_id):
+    return item_basic.get_item_filepath(item_id)
+
+def get_item_date(item_id, add_separator=False):
+    return item_basic.get_item_date(item_id, add_separator=add_separator)
+
+def get_source(item_id):
+    return item_basic.get_source(item_id)
+
+def get_all_sources():
+    return item_basic.get_all_items_sources(r_list=True)
+
+def get_item_basename(item_id):
+    return os.path.basename(item_id)
+
+def get_item_size(item_id):
+    return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
+
+def get_item_encoding(item_id):
+    return None
+
+def get_lines_info(item_id, item_content=None):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    max_length = 0
+    line_id = 0
+    nb_line = 0
+    for line in item_content.splitlines():
+        length = len(line)
+        if length > max_length:
+            max_length = length
+        nb_line += 1
+    return {'nb': nb_line, 'max_length': max_length}
+
+
+def get_item_metadata(item_id, item_content=None):
+    ## TODO: FIXME ##performance
+    # encoding
+    # language
+    # lines info
+    item_metadata = {'date': get_item_date(item_id, add_separator=True),
+                     'source': get_source(item_id),
+                     'size': get_item_size(item_id),
+                     'encoding': get_item_encoding(item_id),
+                     'lines': get_lines_info(item_id, item_content=item_content)
+                     }
+    return item_metadata
+
+def get_item_parent(item_id):
+    return item_basic.get_item_parent(item_id)
+
+def add_item_parent(item_parent, item_id):
+    return item_basic.add_item_parent(item_parent, item_id)
+
+def get_item_content(item_id):
+    return item_basic.get_item_content(item_id)
+
+def get_item_content_html2text(item_id, item_content=None, ignore_links=False):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    h = html2text.HTML2Text()
+    h.ignore_links = ignore_links
+    h.ignore_images = ignore_links
+    return h.handle(item_content)
+
+def remove_all_urls_from_content(item_id, item_content=None):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b'
+    url_regex = re.compile(regex)
+    urls = url_regex.findall(item_content)
+    urls = sorted(urls, key=len, reverse=True)
+    for url in urls:
+        item_content = item_content.replace(url, '')
+
+    regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
+    regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
+    regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
+    re.compile(regex_pgp_public_blocs)
+    re.compile(regex_pgp_signature)
+    re.compile(regex_pgp_message)
+
+    res = re.findall(regex_pgp_public_blocs, item_content)
+    for it in res:
+        item_content = item_content.replace(it, '')
+    res = re.findall(regex_pgp_signature, item_content)
+    for it in res:
+        item_content = item_content.replace(it, '')
+    res = re.findall(regex_pgp_message, item_content)
+    for it in res:
+        item_content = item_content.replace(it, '')
+
+    return item_content
+
+def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7):
+    all_languages = []
+
+    ## CLEAN CONTENT ##
+    content = get_item_content_html2text(item_id, ignore_links=True)
+    content = remove_all_urls_from_content(item_id, item_content=content)
+
+    # REMOVE USELESS SPACE
+    content = ' '.join(content.split())
+    #- CLEAN CONTENT -#
+
+    #print(content)
+    #print(len(content))
+    if len(content) >= min_len:
+        for lang in cld3.get_frequent_languages(content, num_langs=num_langs):
+            if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable:
+                all_languages.append(lang)
+    return all_languages
+
+# API
+def get_item(request_dict):
+    if not request_dict:
+        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+
+    item_id = request_dict.get('id', None)
+    if not item_id:
+        return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
+    if not exist_item(item_id):
+        return {'status': 'error', 'reason': 'Item not found'}, 404
+
+    dict_item = {}
+    dict_item['id'] = item_id
+    date = request_dict.get('date', True)
+    if date:
+        add_separator = False
+        if request_dict.get('date_separator', False):
+            add_separator = True
+        dict_item['date'] = get_item_date(item_id, add_separator=add_separator)
+    tags = request_dict.get('tags', True)
+    if tags:
+        dict_item['tags'] = Tag.get_obj_tag(item_id)
+
+    size = request_dict.get('size', False)
+    if size:
+        dict_item['size'] = get_item_size(item_id)
+
+    content = request_dict.get('content', False)
+    if content:
+        # UTF-8 outpout, # TODO: use base64
+        dict_item['content'] = get_item_content(item_id)
+
+    raw_content = request_dict.get('raw_content', False)
+    if raw_content:
+        dict_item['raw_content'] = get_raw_content(item_id)
+
+    lines_info = request_dict.get('lines', False)
+    if lines_info:
+        dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
+
+    if request_dict.get('pgp'):
+        dict_item['pgp'] = {}
+        if request_dict['pgp'].get('key'):
+            dict_item['pgp']['key'] = get_item_pgp_key(item_id)
+        if request_dict['pgp'].get('mail'):
+            dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
+        if request_dict['pgp'].get('name'):
+            dict_item['pgp']['name'] = get_item_pgp_name(item_id)
+
+    if request_dict.get('cryptocurrency'):
+        dict_item['cryptocurrency'] = {}
+        if request_dict['cryptocurrency'].get('bitcoin'):
+            dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
+
+    return dict_item, 200
+
+
+
+def api_get_item_content_base64_utf8(request_dict):
+    item_id = request_dict.get('id', None)
+    if not request_dict:
+        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+    if not item_id:
+        return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
+    if not exist_item(item_id):
+        return {'status': 'error', 'reason': 'Item not found'}, 404
+
+    item_content = get_item_content(item_id)
+    item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8')
+    return {'status': 'success', 'content': item_content}, 200
+
+
+def api_get_items_sources():
+    item_content = {'sources': get_all_sources()}
+    return item_content, 200
+
+# def check_item_source(request_dict):
+#     source = request_dict.get('source', None)
+#     if not request_dict:
+#         return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+#     if not source:
+#         return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
+#
+#     all_sources = item_basic.get_all_items_sources()
+#
+#     if source not in all_sources:
+#         return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400
+#     return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200
+
+###
+### correlation
+###
+def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False):
+    '''
+    Return all cryptocurrencies of a given item.
+
+    :param item_id: item id
+    :param currencies_type: list of cryptocurrencies type
+    :type currencies_type: list, optional
+    '''
+    return Cryptocurrency.cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
+
+def get_item_pgp(item_id, currencies_type=None, get_nb=False):
+    '''
+    Return all pgp of a given item.
+
+    :param item_id: item id
+    :param currencies_type: list of cryptocurrencies type
+    :type currencies_type: list, optional
+    '''
+    return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
+
+def get_item_username(item_id, sub_type=None, get_nb=False):
+    '''
+    Return all pgp of a given item.
+
+    :param item_id: item id
+    :param sub_type: list of username type
+    :type sub_type: list, optional
+    '''
+    return Username.correlation.get_item_correlation_dict(item_id, correlation_type=sub_type, get_nb=get_nb)
+
+def get_item_decoded(item_id):
+    '''
+    Return all pgp of a given item.
+
+    :param item_id: item id
+    :param currencies_type: list of cryptocurrencies type
+    :type currencies_type: list, optional
+    '''
+    return Decoded.get_item_decoded(item_id)
+
+def get_item_all_screenshot(item_id):
+    '''
+    Return all screenshot of a given item.
+
+    :param item_id: item id
+    '''
+    return Screenshot.get_item_screenshot_list(item_id)
+
+def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
+    '''
+    Retun all correlation of a given item id.
+
+    :param item_id: item id
+    :type domain: str
+
+    :return: a dict of all correlation for a item id
+    :rtype: dict
+    '''
+    if not correlation_names:
+        correlation_names = Correlate_object.get_all_correlation_names()
+    item_correl = {}
+    for correlation_name in correlation_names:
+        if correlation_name=='cryptocurrency':
+            res = get_item_cryptocurrency(item_id, get_nb=get_nb)
+        elif correlation_name=='pgp':
+            res = get_item_pgp(item_id, get_nb=get_nb)
+        elif correlation_name=='username':
+            res = get_item_username(item_id, get_nb=get_nb)
+        elif correlation_name=='decoded':
+            res = get_item_decoded(item_id)
+        elif correlation_name=='screenshot':
+            res = get_item_all_screenshot(item_id)
+        else:
+            res = None
+        # add correllation to dict
+        if res:
+            item_correl[correlation_name] = res
+    return item_correl
+
+
+
+## TODO: REFRACTOR
+def _get_item_correlation(correlation_name, correlation_type, item_id):
+    res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    if res:
+        return list(res)
+    else:
+        return []
+
+## TODO: REFRACTOR
+def get_item_bitcoin(item_id):
+    return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
+
+## TODO: REFRACTOR
+def get_item_pgp_key(item_id):
+    return _get_item_correlation('pgpdump', 'key', item_id)
+
+## TODO: REFRACTOR
+def get_item_pgp_name(item_id):
+    return _get_item_correlation('pgpdump', 'name', item_id)
+
+## TODO: REFRACTOR
+def get_item_pgp_mail(item_id):
+    return _get_item_correlation('pgpdump', 'mail', item_id)
+
+## TODO: REFRACTOR
+def get_item_pgp_correlation(item_id):
+    pass
+
+###
+### GET Internal Module DESC
+###
+def get_item_list_desc(list_item_id):
+    desc_list = []
+    for item_id in list_item_id:
+        desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} )
+    return desc_list
+
+def is_crawled(item_id):
+    return item_basic.is_crawled(item_id)
+
+def get_crawler_matadata(item_id, ltags=None):
+    dict_crawler = {}
+    if is_crawled(item_id):
+        dict_crawler['domain'] = get_item_domain(item_id)
+        if not ltags:
+            ltags = Tag.get_obj_tag(item_id)
+        dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags)
+        dict_crawler['url'] = get_item_link(item_id)
+        dict_crawler['screenshot'] = get_item_screenshot(item_id)
+        dict_crawler['har'] = get_item_har_name(item_id)
+    return dict_crawler
+
+def is_onion(item_id):
+    is_onion = False
+    if len(is_onion) > 62:
+        if is_crawled(item_id) and item_id[-42:-36] == '.onion':
+            is_onion = True
+    return is_onion
+
+def is_item_in_domain(domain, item_id):
+    is_in_domain = False
+    domain_lenght = len(domain)
+    if len(item_id) > (domain_lenght+48):
+        if item_id[-36-domain_lenght:-36] == domain:
+            is_in_domain = True
+    return is_in_domain
+
+def get_item_domain(item_id):
+    return item_basic.get_item_domain(item_id)
+
+def get_domain(item_id):
+    item_id = item_id.split('/')
+    item_id = item_id[-1]
+    return item_id[:-36]
+
+def get_item_domain_with_port(item_id):
+    return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'domain')
+
+def get_item_link(item_id):
+    return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
+
+def get_item_screenshot(item_id):
+    screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
+    if screenshot:
+        return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
+    return ''
+
+def get_item_har_name(item_id):
+    har_path = os.path.join(har_directory, item_id) + '.json'
+    if os.path.isfile(har_path):
+        return har_path
+    else:
+        return None
+
+def get_item_har(har_path):
+    pass
+
+def get_item_filename(item_id):
+    # Creating the full filepath
+    filename = os.path.join(PASTES_FOLDER, item_id)
+    filename = os.path.realpath(filename)
+
+    # incorrect filename
+    if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
+        return None
+    else:
+        return filename
+
+def get_item_duplicate(item_id, r_list=True):
+    res = r_serv_metadata.smembers('dup:{}'.format(item_id))
+    if r_list:
+        if res:
+            return list(res)
+        else:
+            return []
+    return res
+
+def get_item_nb_duplicates(item_id):
+    return r_serv_metadata.scard('dup:{}'.format(item_id))
+
+def get_item_duplicates_dict(item_id):
+    dict_duplicates = {}
+    for duplicate in get_item_duplicate(item_id):
+        duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',')
+        duplicate_id = duplicate[1]
+        if not duplicate_id in dict_duplicates:
+            dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}}
+        algo = duplicate[0]
+        if algo == 'tlsh':
+            similarity = 100 - int(duplicate[2])
+        else:
+            similarity = int(duplicate[2])
+        dict_duplicates[duplicate_id]['algo'][algo] = similarity
+    return dict_duplicates
+
+def add_item_duplicate(item_id, l_dup):
+    for item_dup in l_dup:
+        r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id)
+        r_serv_metadata.sadd('dup:{}'.format(item_id), item_dup)
+
+def delete_item_duplicate(item_id):
+    item_dup = get_item_duplicate(item_id)
+    for item_dup in get_item_duplicate(item_id):
+        r_serv_metadata.srem('dup:{}'.format(item_dup), item_id)
+    r_serv_metadata.delete('dup:{}'.format(item_id))
+
+def get_raw_content(item_id):
+    filepath = get_item_filepath(item_id)
+    with open(filepath, 'rb') as f:
+        file_content = BytesIO(f.read())
+    return file_content
+
+def save_raw_content(item_id, io_content):
+    filepath = get_item_filename(item_id)
+    if os.path.isfile(filepath):
+        #print('File already exist')
+        return False
+    # create subdir
+    dirname = os.path.dirname(filepath)
+    if not os.path.exists(dirname):
+        os.makedirs(dirname)
+    # # TODO: check if is IO file
+    with open(filepath, 'wb') as f:
+        f.write(io_content.getvalue())
+    return True
+
+# IDEA: send item to duplicate ?
+def create_item(obj_id, obj_metadata, io_content):
+    '''
+    Create a new Item (Import or Test only).
+
+    :param obj_id: item id
+    :type obj_metadata: dict - 'first_seen', 'tags'
+
+    :return: is item created
+    :rtype: boolean
+    '''
+    # check if datetime match ??
+
+
+    # # TODO: validate obj_id
+
+    res = save_raw_content(obj_id, io_content)
+    # item saved
+    if res:
+        # creata tags
+        if 'tags' in obj_metadata:
+            # # TODO: handle mixed tags: taxonomies and Galaxies
+            Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type="item")
+        return True
+
+    # Item not created
+    return False
+
+def delete_item(obj_id):
+    # check if item exists
+    if not exist_item(obj_id):
+        return False
+    else:
+        delete_item_duplicate(obj_id)
+        # delete MISP event
+        r_serv_metadata.delete('misp_events:{}'.format(obj_id))
+        r_serv_metadata.delete('hive_cases:{}'.format(obj_id))
+
+        os.remove(get_item_filename(obj_id))
+
+        # get all correlation
+        obj_correlations = get_item_all_correlation(obj_id)
+        for correlation in obj_correlations:
+            if correlation=='cryptocurrency' or correlation=='pgp':
+                for obj2_subtype in obj_correlations[correlation]:
+                    for obj2_id in obj_correlations[correlation][obj2_subtype]:
+                        Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id,
+                                                            obj1_subtype=obj2_subtype)
+            else:
+                for obj2_id in obj_correlations[correlation]:
+                    Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id)
+
+        # delete father/child
+        delete_node(obj_id)
+
+        # delete item metadata
+        r_serv_metadata.delete('paste_metadata:{}'.format(obj_id))
+
+        return True
+
+    ### TODO in inport V2
+    # delete from tracked items
+
+    # # # TODO: # FIXME: LATER
+    # delete from queue
+    ###
+    return False
+
+#### ####
+def delete_node(item_id):
+    if is_node(item_id):
+        if is_crawled(item_id):
+            delete_domain_node(item_id)
+        item_basic._delete_node(item_id)
+
+def delete_domain_node(item_id):
+    if is_domain_root(item_id):
+        # remove from domain history
+        domain, port = get_item_domain_with_port(item_id).split(':')
+        domain_basic.delete_domain_item_core(item_id, domain, port)
+    for child_id in get_all_domain_node_by_item_id(item_id):
+        delete_item(child_id)
+
+
+#if __name__ == '__main__':

bin/lib/objects/Pgps.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Pgp(AbstractObject):
+    """
+    AIL Pgp Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(Pgp, self).__init__('pgp', id, subtype=subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        if self.subtype == 'key':
+            icon = '\uf084'
+        elif self.subtype == 'name':
+            icon = '\uf507'
+        elif self.subtype == 'mail':
+            icon = '\uf1fa'
+        else:
+            icon = 'times'
+        return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/Screenshots.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+HASH_DIR = config_loader.get_config_str('Directories', 'hash')
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Screenshot(AbstractObject):
+    """
+    AIL Screenshot Object. (strings)
+    """
+
+    def __init__(self, id):
+        super(Screenshot, self).__init__('screenshot', id)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', object_type=self.type, correlation_id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        return {'style': 'fas', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/Usernames.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+
+# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+from abstract_object import AbstractObject
+
+config_loader = ConfigLoader.ConfigLoader()
+
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Username(AbstractObject):
+    """
+    AIL Username Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(Username, self).__init__('username', id, subtype=subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        if self.subtype == 'telegram':
+            style = 'fab'
+            icon = '\uf2c6'
+        elif self.subtype == 'twitter':
+            style = 'fab'
+            icon = '\uf099'
+        else:
+            style = 'fas'
+            icon = '\uf007'
+        return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
+
+    ############################################################################
+    ############################################################################
+    ############################################################################
+
+    def exist_correlation(self):
+        pass
+
+    ############################################################################
+    ############################################################################
+
+
+
+#if __name__ == '__main__':

bin/lib/objects/abstract_object.py

@@ -0,0 +1,135 @@
+# -*-coding:UTF-8 -*
+"""
+Base Class for AIL Objects
+"""
+
+##################################
+# Import External packages
+##################################
+import os
+import sys
+from abc import ABC, abstractmethod
+
+#from flask import url_for
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from packages import Tag
+from lib.Investigations import is_object_investigated, get_obj_investigations
+
+# # TODO: ADD CORRELATION ENGINE
+
+class AbstractObject(ABC):
+    """
+    Abstract Object
+    """
+
+    # first seen last/seen ??
+    # # TODO: - tags
+    #         - handle + refactor coorelations
+    #         - creates others objects
+
+    def __init__(self, obj_type, id, subtype=None):
+        """ Abstract for all the AIL object
+
+        :param obj_type: object type (item, ...)
+        :param id: Object ID
+        """
+        self.id = id
+        self.type = obj_type
+        self.subtype = subtype
+
+    def get_id(self):
+        return self.id
+
+    def get_type(self):
+        return self.type
+
+    def get_subtype(self, r_str=False):
+        if not self.subtype:
+            if r_str:
+                return ''
+        return self.subtype
+
+    def get_default_meta(self, tags=False):
+        dict_meta = {'id': self.get_id(),
+                     'type': self.get_type(),
+                     'subtype': self.get_subtype()}
+        if tags:
+            dict_meta['tags'] = self.get_tags()
+        return dict_meta
+
+    ## Tags ##
+    def get_tags(self, r_set=False):
+        tags = Tag.get_obj_tag(self.id)
+        if r_set:
+            tags = set(tags)
+        return tags
+
+    ## ADD TAGS ????
+    #def add_tags(self):
+
+    #- Tags -#
+
+    ## Investigations ##
+    # # TODO: unregister =====
+
+    def is_investigated(self):
+        if not self.subtype:
+            is_investigated = is_object_investigated(self.id, self.type)
+        else:
+            is_investigated = is_object_investigated(self.id, self.type, self.subtype)
+        return is_investigated
+
+    def get_investigations(self):
+        if not self.subtype:
+            investigations = get_obj_investigations(self.id, self.type)
+        else:
+            investigations = get_obj_investigations(self.id, self.type, self.subtype)
+        return investigations
+    #- Investigations -#
+
+    def _delete(self):
+        # DELETE TAGS
+        Tag.delete_obj_all_tags(self.id, self.type)
+        # # TODO: remove from investigations
+
+    @abstractmethod
+    def delete(self):
+        """
+        Delete Object: used for the Data Retention
+        """
+        pass
+
+    # @abstractmethod
+    # def get_meta(self):
+    #     """
+    #     get Object metadata
+    #     """
+    #     pass
+
+    @abstractmethod
+    def get_svg_icon(self):
+        """
+        Get object svg icon
+        """
+        pass
+
+    @abstractmethod
+    def get_link(self, flask_context=False):
+        pass
+
+    # # TODO:
+    # @abstractmethod
+    # def get_correlations(self, message):
+    #     """
+    #     Get object correlations
+    #     """
+    #     pass
+
+
+    # # TODO: get favicon
+    # # TODO: get url
+    # # TODO: get metadata

bin/lib/objects/ail_objects.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import uuid
+import redis
+
+from abc import ABC
+from flask import url_for
+
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/objects'))
+from Decodeds import Decoded
+from Domains import Domain
+from CryptoCurrencies import CryptoCurrency
+from Items import Item
+from Pgps import Pgp
+from Screenshots import Screenshot
+from Usernames import Username
+
+##################################################################
+##################################################################
+#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+
+#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+##################################################################
+##################################################################
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
+
+class AILObjects(object):
+    initial = 0
+    ongoing = 1
+    completed = 2
+
+
+# # TODO: check change paste => item
+def get_all_objects():
+    return ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
+
+def get_object(obj_type, subtype, id):
+    if obj_type == 'item':
+        return Item(id)
+    elif obj_type == 'domain':
+        return Domain(id)
+    elif obj_type == 'decoded':
+        return Decoded(id)
+    elif obj_type == 'screenshot':
+        return Screenshot(id)
+    elif obj_type == 'cryptocurrency':
+        return CryptoCurrency(id, subtype)
+    elif obj_type == 'pgp':
+        return Pgp(id, subtype)
+    elif obj_type == 'username':
+        return Username(id, subtype)
+
+def get_object_svg(obj_type, subtype, id):
+    object = get_object(obj_type, subtype, id)
+    return object.get_svg_icon()
+
+def get_objects_meta(l_dict_objs, icon=False, url=False, flask_context=False):
+    l_meta = []
+    for dict_obj in l_dict_objs:
+        object = get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id'])
+        dict_meta = object.get_default_meta(tags=True)
+        if icon:
+            dict_meta['icon'] = object.get_svg_icon()
+        if url:
+            dict_meta['link'] = object.get_link(flask_context=flask_context)
+        l_meta.append(dict_meta)
+    return l_meta
+
+
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################

bin/packages/Item.py

@@ -26,7 +26,7 @@ import Decoded
 import Screenshot
 import Username
 
-from ail_objects import AbstractObject
+from objects.abstract_object import AbstractObject
 from item_basic import *
 
 config_loader = ConfigLoader.ConfigLoader()

configs/core.cfg.sample

@@ -73,9 +73,6 @@ criticalNumberToAlert=8
 #Will be considered as false positive if less that X matches from the top password list
 minTopPassList=5
 
-[Curve]
-max_execution_time = 90
-
 [Onion]
 save_i2p = False
 max_execution_time = 180
@@ -149,31 +146,27 @@ port = 6381
 db = 1
 
 ##### ARDB #####
-[ARDB_Curve]
+
+[ARDB_DB]
 host = localhost
 port = 6382
-db = 1
+db = 0
+
+[DB_Tracking]
+host = localhost
+port = 6382
+db = 3
 
 [ARDB_Sentiment]
 host = localhost
 port = 6382
 db = 4
 
-[ARDB_TermFreq]
-host = localhost
-port = 6382
-db = 2
-
 [ARDB_TermCred]
 host = localhost
 port = 6382
 db = 5
 
-[ARDB_DB]
-host = localhost
-port = 6382
-db = 0
-
 [ARDB_Trending]
 host = localhost
 port = 6382

update/v4.1/Update.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
+from ail_updater import AIL_Updater
+
+class Updater(AIL_Updater):
+    """default Updater."""
+
+    def __init__(self, version):
+        super(Updater, self).__init__(version)
+
+    def update(self):
+        config_loader = ConfigLoader.ConfigLoader()
+        r_tracking = config_loader.get_redis_conn("DB_Tracking")
+        config_loader = None
+
+        # FLUSH OLD DB
+        r_tracking.flushdb()
+
+if __name__ == '__main__':
+
+    updater = Updater('v4.1')
+
+
+    updater.run_update()

update/v4.1/Update.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+# SUBMODULES #
+git submodule update
+
+exit 0

var/www/Flask_server.py

@@ -42,6 +42,7 @@ from blueprints.crawler_splash import crawler_splash
 from blueprints.correlation import correlation
 from blueprints.tags_ui import tags_ui
 from blueprints.import_export import import_export
+from blueprints.investigations_b import investigations_b
 from blueprints.objects_item import objects_item
 from blueprints.hunters import hunters
 from blueprints.old_endpoints import old_endpoints
@@ -101,6 +102,7 @@ app.register_blueprint(crawler_splash, url_prefix=baseUrl)
 app.register_blueprint(correlation, url_prefix=baseUrl)
 app.register_blueprint(tags_ui, url_prefix=baseUrl)
 app.register_blueprint(import_export, url_prefix=baseUrl)
+app.register_blueprint(investigations_b, url_prefix=baseUrl)
 app.register_blueprint(objects_item, url_prefix=baseUrl)
 app.register_blueprint(hunters, url_prefix=baseUrl)
 app.register_blueprint(old_endpoints, url_prefix=baseUrl)

var/www/blueprints/crawler_splash.py

@@ -28,7 +28,7 @@ import crawlers
 import Domain
 import Language
 
-import Config_DB
+#import Config_DB
 
 r_cache = Flask_config.r_cache
 r_serv_db = Flask_config.r_serv_db
@@ -295,6 +295,15 @@ def domains_search_name():
                                 l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label,
                                 domains_types=domains_types)
 
+@crawler_splash.route('/domains/TODO', methods=['GET'])
+@login_required
+@login_analyst
+def domains_todo():
+    domain_type = request.args.get('type')
+    last_domains = Domain.get_last_crawled_domains(domain_type)
+
+
+
 ##--  --##
 
 
@@ -349,8 +358,8 @@ def crawler_cookiejar_add_post():
     return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
 
 @crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
-#@login_required
-#@login_read_only
+@login_required
+@login_read_only
 def crawler_cookiejar_all():
     user_id = current_user.get_id()
     user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id))
@@ -358,8 +367,8 @@ def crawler_cookiejar_all():
     return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar)
 
 @crawler_splash.route('/crawler/cookiejar/show', methods=['GET'])
-#@login_required
-#@login_read_only
+@login_required
+@login_read_only
 def crawler_cookiejar_show():
     user_id = current_user.get_id()
     cookiejar_uuid = request.args.get('cookiejar_uuid')
@@ -379,8 +388,8 @@ def crawler_cookiejar_show():
                                         l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid)
 
 @crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET'])
-#@login_required
-#@login_read_only
+@login_required
+@login_read_only
 def crawler_cookiejar_cookie_delete():
     user_id = current_user.get_id()
     cookiejar_uuid = request.args.get('cookiejar_uuid')
@@ -392,8 +401,8 @@ def crawler_cookiejar_cookie_delete():
     return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
 
 @crawler_splash.route('/crawler/cookiejar/delete', methods=['GET'])
-#@login_required
-#@login_read_only
+@login_required
+@login_read_only
 def crawler_cookiejar_delete():
     user_id = current_user.get_id()
     cookiejar_uuid = request.args.get('cookiejar_uuid')

var/www/blueprints/import_export.py

@@ -182,6 +182,20 @@ def add_object_id_to_export():
     # redirect
     return redirect(url_for('import_export.export_object'))
 
+@import_export.route("/import_export/investigation", methods=['GET'])
+@login_required
+@login_analyst
+def export_investigation():
+    investigation_uuid = request.args.get("uuid")
+
+    if MispExport.ping_misp():
+        event_metadata = MispExport.create_investigation_event(investigation_uuid)
+    else:
+        return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400 
+
+    return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
+
+
 # @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
 # @login_required
 # @login_analyst

var/www/blueprints/investigations_b.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: ail_investigations
+'''
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append('modules')
+import Flask_config
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import Investigations
+from lib.objects import ail_objects
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Tag
+
+# ============ BLUEPRINT ============
+investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations'))
+
+# ============ VARIABLES ============
+bootstrap_label = Flask_config.bootstrap_label
+
+# ============ FUNCTIONS ============
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
+# ============= ROUTES ==============
+
+@investigations_b.route("/investigations", methods=['GET'])
+@login_required
+@login_read_only
+def investigations_dashboard():
+    investigations = Investigations.get_all_investigations_meta(r_str=True)
+    return render_template("investigations.html", bootstrap_label=bootstrap_label,
+                                investigations=investigations)
+
+
+@investigations_b.route("/investigation", methods=['GET']) ## FIXME: add /view ????
+@login_required
+@login_read_only
+def show_investigation():
+    investigation_uuid = request.args.get("uuid")
+    investigation = Investigations.Investigation(investigation_uuid)
+    metadata = investigation.get_metadata(r_str=True)
+    objs = ail_objects.get_objects_meta(investigation.get_objects(), icon=True)
+    return render_template("view_investigation.html", bootstrap_label=bootstrap_label,
+                                metadata=metadata, investigation_objs=objs)
+
+
+@investigations_b.route("/investigation/add", methods=['GET', 'POST'])
+@login_required
+@login_read_only
+def add_investigation():
+    if request.method == 'POST':
+        user_id = current_user.get_id()
+        name = request.form.get("investigation_name")
+        date = request.form.get("investigation_date")
+        threat_level = request.form.get("threat_level")
+        analysis = request.form.get("analysis")
+        info = request.form.get("investigation_info")
+        # tags
+        taxonomies_tags = request.form.get('taxonomies_tags')
+        if taxonomies_tags:
+            try:
+                taxonomies_tags = json.loads(taxonomies_tags)
+            except Exception:
+                taxonomies_tags = []
+        else:
+            taxonomies_tags = []
+        galaxies_tags = request.form.get('galaxies_tags')
+        if galaxies_tags:
+            try:
+                galaxies_tags = json.loads(galaxies_tags)
+            except Exception:
+                galaxies_tags = []
+        tags = taxonomies_tags + galaxies_tags
+
+        input_dict = {"user_id": user_id, "name": name,
+                      "threat_level": threat_level, "date": date,
+                      "analysis": analysis, "info": info, "tags": tags}
+        res = Investigations.api_add_investigation(input_dict)
+        if res[1] != 200:
+            return create_json_response(res[0], res[1])
+
+        return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
+    else:
+        return render_template("add_investigation.html", tags_selector_data=Tag.get_tags_selector_data())
+
+
+@investigations_b.route("/investigation/edit", methods=['GET', 'POST'])
+@login_required
+@login_read_only
+def edit_investigation():
+    if request.method == 'POST':
+        user_id = current_user.get_id()
+        investigation_uuid = request.form.get("investigation_uuid")
+        name = request.form.get("investigation_name")
+        date = request.form.get("investigation_date")
+        threat_level = request.form.get("threat_level")
+        analysis = request.form.get("analysis")
+        info = request.form.get("investigation_info")
+
+        # tags
+        taxonomies_tags = request.form.get('taxonomies_tags')
+        if taxonomies_tags:
+            try:
+                taxonomies_tags = json.loads(taxonomies_tags)
+            except Exception:
+                taxonomies_tags = []
+        else:
+            taxonomies_tags = []
+        galaxies_tags = request.form.get('galaxies_tags')
+        if galaxies_tags:
+            try:
+                galaxies_tags = json.loads(galaxies_tags)
+            except Exception:
+                galaxies_tags = []
+        tags = taxonomies_tags + galaxies_tags
+
+        input_dict = {"user_id": user_id, "uuid": investigation_uuid,
+                      "name": name, "threat_level": threat_level,
+                      "analysis": analysis, "info": info, "tags": tags}
+        res = Investigations.api_edit_investigation(input_dict)
+        if res[1] != 200:
+            return create_json_response(res[0], res[1])
+
+        return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
+    else:
+        investigation_uuid = request.args.get('uuid')
+        investigation = Investigations.Investigation(investigation_uuid)
+        metadata = investigation.get_metadata(r_str=False)
+        taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(metadata['tags'])
+        tags_selector_data = Tag.get_tags_selector_data()
+        tags_selector_data['taxonomies_tags'] = taxonomies_tags
+        tags_selector_data['galaxies_tags'] = galaxies_tags
+        return render_template("add_investigation.html", edit=True,
+                                tags_selector_data=tags_selector_data, metadata=metadata)
+
+@investigations_b.route("/investigation/delete", methods=['GET'])
+@login_required
+@login_read_only
+def delete_investigation():
+    investigation_uuid = request.args.get('uuid')
+    input_dict = {"uuid": investigation_uuid}
+    res = Investigations.api_delete_investigation(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('investigations_b.investigations_dashboard'))
+
+@investigations_b.route("/investigation/object/register", methods=['GET'])
+@login_required
+@login_read_only
+def register_investigation():
+    investigations_uuid = request.args.get('uuids')
+    investigations_uuid = investigations_uuid.split(',')
+
+    object_type = request.args.get('type')
+    object_subtype = request.args.get('subtype')
+    object_id = request.args.get('id')
+
+    for investigation_uuid in investigations_uuid:
+        input_dict = {"uuid": investigation_uuid, "id": object_id,
+                      "type": object_type, "subtype": object_subtype}
+        res = Investigations.api_register_object(input_dict)
+        if res[1] != 200:
+            return create_json_response(res[0], res[1])
+    return redirect(url_for('investigations_b.investigations_dashboard', uuid=investigation_uuid))
+
+@investigations_b.route("/investigation/object/unregister", methods=['GET'])
+@login_required
+@login_read_only
+def unregister_investigation():
+    investigation_uuid = request.args.get('uuid')
+    object_type = request.args.get('type')
+    object_subtype = request.args.get('subtype')
+    object_id = request.args.get('id')
+    input_dict = {"uuid": investigation_uuid, "id": object_id,
+                  "type": object_type, "subtype": object_subtype}
+    res = Investigations.api_unregister_object(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
+
+
+@investigations_b.route("/investigation/all/selector_json")
+@login_required
+@login_read_only
+def get_investigations_selector_json():
+    return jsonify(Investigations.get_investigations_selector())
+
+
+#
+# @investigations_b.route("/object/item") #completely shows the paste in a new tab
+# @login_required
+# @login_analyst
+# def showItem(): # # TODO: support post
+#     item_id = request.args.get('id')
+#     if not item_id or not Item.exist_item(item_id):
+#         abort(404)
+#
+#     return render_template("show_item.html", bootstrap_label=bootstrap_label)

var/www/modules/settings/Flask_settings.py

@@ -288,5 +288,11 @@ def passive_dns_change_state():
     passivedns_enabled = d4.change_passive_dns_state(new_state)
     return redirect(url_for('settings.passive_dns'))
 
+@settings.route("/settings/ail", methods=['GET'])
+@login_required
+@login_admin
+def ail_configs():
+    return render_template("ail_configs.html", passivedns_enabled=None)
+
 # ========= REGISTRATION =========
 app.register_blueprint(settings, url_prefix=baseUrl)

var/www/modules/settings/templates/ail_configs.html

@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Passive DNS - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'settings/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+        <div class="card my-3">
+          <div class="card-body">
+
+            <div class="container-fluid">
+          		<div class="row">
+                <div class="col-12 col-lg-4">
+                  <a href="https://www.misp-project.org/">
+                    <img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP project">
+                  </a>
+                </div>
+                <div class="col-12 col-lg-8">
+                  <form>
+                    <div class="form-group">
+                      <label for="misp_url">Url</label>
+                      <input class="form-control" type="text" id="misp_url" name="misp_url" placeholder="URL of the MISP instance">
+                    </div>
+                    <div class="form-group">
+                      <label for="misp_key">Key</label>
+                      <input class="form-control" type="text" id="misp_key" name="misp_key" placeholder="API key of the user you want to use">
+                    </div>
+                    <div class="form-group">
+                      <label for="misp_key">Org UUID</label>
+                      <input class="form-control" type="text" id="org_uuid" name="org_uuid" placeholder="Org UUID">
+                    </div>
+                    <button type="submit" class="btn btn-primary">Update</button>
+                  </form>
+                </div>
+              </div>
+            </div>
+
+          </div>
+        </div>
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$("#nav_settings").removeClass("text-muted");
+  $("#nav_ail_configs").addClass("active");
+} );
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+</html>

var/www/static/css/ail-project.css

@@ -5,3 +5,16 @@
     padding-left: 0.15em;
     background-color: #2e5;
 }
+
+.blue {
+    color: #0088cc !important;
+}
+
+.bold {
+    font-weight: bold;
+}
+
+.object_node_icon {
+    font-size: 16px;
+    pointer-events: none;
+}

var/www/templates/correlation/metadata_card_cryptocurrency.html

@@ -78,6 +78,14 @@
 				<a class="btn btn-secondary" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&type_id=bitcoin&correlation_id={{ dict_object['correlation_id'] }}&expand_card=True&correlation_objects=paste">Expand Bitcoin address</a>
 			{% endif %}
 		{% endif %}
+
+		{% with obj_type='cryptocurrency', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>
 

var/www/templates/correlation/metadata_card_decoded.html

@@ -88,6 +88,14 @@
 			<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
 			</button>
 		</a>
+
+		{% with obj_type='decoded', obj_id=dict_object['correlation_id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>
 

var/www/templates/correlation/metadata_card_domain.html

@@ -64,5 +64,13 @@
 
 			</li>
 		</ul>
+
+		{% with obj_type='domain', obj_id=dict_object['correlation_id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>

var/www/templates/correlation/metadata_card_paste.html

@@ -46,5 +46,13 @@
 
 			</li>
 		</ul>
+
+		{% with obj_type='item', obj_id=dict_object['correlation_id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>

var/www/templates/correlation/metadata_card_pgp.html

@@ -40,6 +40,14 @@
 				</div>
 			</li>
 		</ul>
+
+		{% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>
 

var/www/templates/correlation/metadata_card_screenshot.html

@@ -78,6 +78,14 @@
 				</div>
 			</li>
 		</ul>
+
+		{% with obj_type='screenshot', obj_id=dict_object['correlation_id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>
 

var/www/templates/correlation/metadata_card_username.html

@@ -40,6 +40,14 @@
 				</div>
 			</li>
 		</ul>
+
+		{% with obj_type='username', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
 	</div>
 </div>
 

var/www/templates/investigations/add_investigation.html

@@ -0,0 +1,214 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/ail-project.css') }}" />
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+						<div class="card my-3">
+						  <div class="card-header bg-dark text-white">
+								<h5 class="card-title">
+									{% if edit %}
+										Edit Investigation
+									{% else %}
+										Create Investigation
+									{% endif %}
+								</h5>
+							</div>
+						  <div class="card-body">
+
+								<form action="{% if edit %}{{ url_for('investigations_b.edit_investigation') }}{% else %}{{ url_for('investigations_b.add_investigation') }}{% endif %}" method='post' onsubmit="SubmitCreateInvestigation();">
+
+									{% if edit %}
+										<input id="investigation_uuid" name="investigation_uuid" type="text" value="{{ metadata['uuid'] }}" hidden>
+									{% endif %}
+
+									<div class="row">
+										<div class="col-12 col-xl-9">
+											<div class="input-group mb-2 mr-sm-2">
+										    <div class="input-group-prepend">
+										      <div class="input-group-text bg-dark text-white"><i class="fas fa-quote-right"></i></div>
+										    </div>
+												<input id="investigation_info" name="investigation_info" class="form-control" placeholder="Quick Investigation Info" type="text" {% if edit %}value="{{metadata['info']}}"{% endif %} required>
+										  </div>
+
+											<div class="row">
+												<div class="col-12 col-xl-6">
+													<div class="form-group">
+													 <label for="threat_level">Threat Level:
+														 <span id="threat_level_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
+														 <script type="text/javascript">
+														 		$(function() {
+																 		$('#threat_level_idInfoPopover').popover({
+																				html: true,
+																				content: function() {
+																				 		var tempSelector = '#threat_level';
+																						return $('<div>').append(
+												                        $('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
+												                    ).append(
+												                        $('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
+												                    );
+																				}
+																		});
+																		var fieldDesc = {"1":"*high* means sophisticated APT malware or 0-day attack","2":"*medium* means APT malware","3":"*low* means mass-malware","4":"*undefined* no risk"};
+																});
+														 </script>
+													 </label>
+													 <select class="form-control" id="threat_level" name="threat_level" required>
+														 <option value="1">High</option>
+														 <option value="2">Medium</option>
+														 <option value="3">Low</option>
+														 <option value="4">Undefined</option>
+													 </select>
+													</div>
+												</div>
+												<div class="col-12 col-xl-6">
+													<div class="form-group">
+													 <label for="analysis">Analysis:
+														 <span id="analysis_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
+														 <script type="text/javascript">
+														 		$(function() {
+																		$('#analysis_idInfoPopover').popover({
+																				html: true,
+																				content: function() {
+																						var tempSelector = '#analysis';
+																						return $('<div>').append(
+													                  		$('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
+													                  ).append(
+													                  		$('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
+													                  );
+																				}
+																		});
+																		var fieldDesc = {"0":"Investigation has just been created and is in an initial state","1":"The analysis is still ongoing","2":"The Investigation creator considers the analysis complete"};
+																});
+														 </script>
+														</label>
+													 <select class="form-control" id="analysis" name="analysis" required>
+														 <option value="0">Initial</option>
+														 <option value="1">Ongoing</option>
+														 <option value="2">Completed</option>
+													 </select>
+													</div>
+												</div>
+											</div>
+
+											<div class="input-group" id="ivestigation-date">
+												<div class="input-group-prepend"><span class="input-group-text bg-secondary text-white"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+												<input class="form-control" id="ivestigation-date-input" placeholder="yyyy-mm-dd" name="investigation_date" autocomplete="off" required>
+											</div>
+
+											<div class="card my-4">
+											  <div class="card-header bg-secondary text-white">
+													<b>Tags</b>
+												</div>
+											  <div class="card-body">
+													{% include 'tags/block_tags_selector.html' %}
+												</div>
+											</div>
+
+										</div>
+
+										<div class="col-12 col-xl-3">
+											{% if edit %}
+												Edit Investigation
+											{% else %}
+												Create a new Investigation
+											{% endif %}
+										</div>
+									</div>
+
+
+									<br>
+									<button class="btn btn-primary mt-2">
+										{% if edit %}
+											<i class="fas fa-pencil-alt"></i> Edit Investigation
+										{% else %}
+											<i class="fas fa-plus"></i> Create Investigation
+										{% endif %}
+									</button>
+
+								</form>
+
+						  </div>
+						</div>
+
+
+			</div>
+
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$('#nav_investigation').removeClass("text-muted");
+	$("#nav_add_investigation").addClass("active");
+
+	{% if edit %}
+		var investigation_date = "{{metadata['date']}}";
+	{% else %}
+		var investigation_date = getTodayDate();
+	{% endif %}
+	$('#ivestigation-date-input').val(investigation_date);
+	$('#ivestigation-date-input').dateRangePicker({
+			autoClose: true,
+			singleDate : true,
+			showShortcuts: false,
+			startDate: false,
+	});
+
+	{% if edit %}
+		$('#threat_level').val({{metadata['threat_level']}});
+		$('#analysis').val({{metadata['analysis']}});
+	{% endif %}
+
+});
+
+function SubmitCreateInvestigation() {
+	var tags = ltags.getValue();
+	var tagsgalaxy = ltagsgalaxies.getValue();
+	$('#ltags').val(tags);
+	$('#ltagsgalaxies').val(tagsgalaxy);
+	return true;
+}
+
+function getTodayDate() {
+	var date = new Date();
+	var month = date.getMonth()+1;
+	var day = date.getDate();
+	return date.getFullYear() +'-'+ (month<10 ? '0' : '') + month +'-'+ (day<10 ? '0' : '') + day;
+}
+
+
+</script>

var/www/templates/investigations/investigations.html

@@ -0,0 +1,90 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>Investigations</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<h3 class="mt-2 text-secondary">
+					<i class="fas fa-microscope"></i> Investigations:
+				</h3>
+
+				<table id="table_investigation" class="table table-striped border-primary">
+          <thead class="bg-dark text-white">
+            <tr>
+								<th>Name</th>
+                <th>Date</th>
+								<th>last modified</th>
+								<td>Info</td>
+                <th>Nb Objects</th>
+            </tr>
+          </thead>
+          <tbody style="font-size: 15px;">
+            {% for dict_investigation in investigations %}
+              <tr class="border-color: blue;">
+								<td>
+									<a href="{{ url_for('investigations_b.show_investigation') }}?uuid={{ dict_investigation['uuid'] }}">
+										{{ dict_investigation['info']}}
+										<div>
+											{% for tag in dict_investigation['tags'] %}
+												<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+											{% endfor %}
+										</div>
+									</a>
+								</td>
+								<td>{{ dict_investigation['date']}}</td>
+								<td>{{ dict_investigation['last_change']}}</td>
+								<td>{{ dict_investigation['info']}}</td>
+								<td>{{ dict_investigation['nb_objects']}}</td>
+              </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+
+			</div>
+
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$('#nav_sync').removeClass("text-muted");
+	$("#navsync_queues").addClass("active");
+
+  $('#table_investigation').DataTable({
+        "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+        "iDisplayLength": 10,
+        "order": [[ 2, "desc" ]]
+  });
+
+});
+
+
+</script>

var/www/templates/investigations/view_investigation.html

@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="card my-1">
+					<div class="card-header bg-dark text-white">
+						<h4 class="card-title">{{metadata['info']}}</h4>
+					</div>
+					<div class="card-body">
+						<div class="container-fluid">
+							<div class="row">
+								<div class="col-12 col-lg-6">
+
+									<table class="table table-hover">
+				            <tr>
+				              <th style="width:30%">UUID</th>
+				              <td>{{metadata['uuid']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Creator</th>
+				              <td>{{metadata['user_creator']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Tags</th>
+				              <td>
+												{% for tag in metadata['tags'] %}
+													<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+												{% endfor %}
+											</td>
+				            </tr>
+				            <tr>
+				              <th>Date</th>
+				              <td>{{metadata['date']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Threat Level</th>
+				              <td>{{metadata['threat_level']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Analysis</th>
+				              <td>{{metadata['analysis']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Info</th>
+				              <td>{{metadata['info']}}</td>
+				            </tr>
+										<tr>
+				              <th># Objects</th>
+				              <td>{{metadata['nb_objects']}}</td>
+				            </tr>
+				            <tr>
+				              <th>Timestamp</th>
+				              <td>{{metadata['timestamp']}}</td>
+				            </tr>
+										<tr>
+				              <th>Last change</th>
+				              <td>{{metadata['last_change']}}</td>
+				            </tr>
+				          </table>
+
+								</div>
+								<div class="col-12 col-lg-6">
+
+									<div class="my-4">
+										<a href="{{ url_for('investigations_b.delete_investigation') }}?uuid={{metadata['uuid']}}">
+											<button type="button" class="btn btn-danger">
+												<i class="fas fa-trash-alt"></i> <b>Delete</b>
+												</button>
+										</a>
+										<a href="{{ url_for('investigations_b.edit_investigation') }}?uuid={{metadata['uuid']}}">
+											<button type="button" class="btn btn-info">
+												<i class="fas fa-pencil-alt"></i> <b>Edit</b>
+											</button>
+										</a>
+										<a class="btn btn-outline-dark" target="_blank" href="{{ url_for('import_export.export_investigation')}}?uuid={{metadata['uuid']}}">
+											<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
+											Export as Event
+										</a>
+									</div>
+
+								</div>
+							</div>
+						</div>
+
+					</div>
+				</div>
+
+				<h3>Objects</h3>
+
+				<table id="table_sync_queues" class="table table-striped border-primary">
+			  	<thead class="bg-dark text-white">
+			    	<tr>
+							<th>Type</th>
+			      	<th></th>
+							<th>Id</th>
+				     	<th>Tags</th>
+							<th></th>
+				   	</tr>
+					</thead>
+					<tbody style="font-size: 15px;">
+						{% for object in investigation_objs %}
+							<tr class="border-color: blue;">
+								<td>
+									{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
+										{% include 'objects/obj_svg_block.html' %}
+									{% endwith %}
+									{{ object['type']}}
+								</td>
+								<td>
+									{% if object['subtype'] %}
+										{{ object['subtype']}}
+									{% endif %}
+								</td>
+								<td>{{ object['id']}}</td>
+								<td>
+									{% for tag in object['tags'] %}
+										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+									{% endfor %}
+								</td>
+								<td class="text-right">
+									<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">
+										<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>
+									</a>
+								</td>
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+
+
+
+		</div>
+
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$('#nav_sync').removeClass("text-muted");
+
+  $('#table_sync_queues').DataTable({
+        "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+        "iDisplayLength": 10,
+        "order": [[ 0, "asc" ]]
+  });
+
+});
+
+
+</script>

var/www/templates/modals/investigations_register_obj.html

@@ -0,0 +1,62 @@
+<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+<div id="investigations_register_obj_modal" class="modal fade" role="dialog">
+  <div class="modal-dialog modal-lg">
+
+    <div id="investigations_register_obj_content" class="modal-content">
+      <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
+        <h4>Add to Investigations</h4>
+      </div>
+
+      <div class="modal-body">
+
+        <div class="input-group" >
+          <input id="linvestigations" type="text" class="form-control" autocomplete="off" style="width: 760px">
+        </div>
+
+
+      </div>
+
+      <div class="modal-footer">
+        <a class="btn btn-info" href="{{ url_for('investigations_b.add_investigation') }}" target="_blank">
+            <i class="fas fa-microscope"></i>
+            <span class="label-icon">Create Investigation </span>
+        </a>
+        <button class="btn btn-primary" onclick="Register_Obj()">
+            <i class="fas fa-plus"></i>
+            Add to Investigations
+        </button>
+        <button type="button" class="btn btn-outline-dark" data-dismiss="modal" >Close</button>
+
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+var linvestigations;
+
+$('#investigations_register_obj_modal').on('shown.bs.modal', function () {
+
+  $.getJSON("{{ url_for('investigations_b.get_investigations_selector_json') }}",
+      function(data) {
+        console.log(data);
+
+        linvestigations = $('#linvestigations').tagSuggest({
+          data: data,
+          maxDropHeight: 200,
+          name: 'linvestigations',
+          emptyText: 'Select Investigations'
+        });
+      }
+  );
+
+});
+
+function Register_Obj() {
+  var uuids = linvestigations.getValue();
+  // TODO: REQUEST
+  window.location.replace("{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}");
+}
+</script>

var/www/templates/objects/item/show_item.html

@@ -97,6 +97,16 @@
 						</button>
 					</a>
         </div>
+        <div>
+          {% with obj_type='item', obj_id=dict_item['id'], obj_subtype=''%}
+            {% include 'modals/investigations_register_obj.html' %}
+          {% endwith %}
+          <div class="mr-2">
+            <button type="button" class="btn btn-lg btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+              <i class="fas fa-microscope"></i> Investigations
+            </button>
+          </div>
+        </div>
         <div class="mx-2">
           {% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%}
             {% include 'import_export/block_add_user_object_to_export.html' %}

var/www/templates/objects/obj_svg_block.html

@@ -0,0 +1,6 @@
+<svg height="26" width="26">
+	<g class="nodes">
+		<circle cx="13" cy="13" r="13" fill="{{ color }}"></circle>
+		<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="object_node_icon {{ style }}">{{ icon }}</text>
+	</g>
+</svg>

var/www/templates/settings/menu_sidebar.html

@@ -44,7 +44,13 @@
       		  <span>Settings</span>
       		</h5>
           <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
-      	    <li class="nav-item">
+            <!-- <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('settings.ail_configs')}}" id="nav_ail_configs">
+      	        <img src="{{ url_for('static', filename='image/ail.png')}}" alt="AIL servers" style="width:25px;">
+      	        <span>AIL Configs</span>
+      	      </a>
+      	    </li> -->
+            <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
       	        <img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
       	        <span>Passive DNS</span>

var/www/templates/sidebars/sidebar_objects.html

@@ -0,0 +1,94 @@
+      <div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
+
+      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
+          <i class="fas fa-align-left"></i>
+      	  <span>Toggle Sidebar</span>
+      	</button>
+
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_investigation">
+          <h5 class="d-flex text-muted w-100">
+      		  <span>Investigations</span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('investigations_b.investigations_dashboard')}}" id="nav_investigation_dashboard">
+                <i class="fas fa-microscope"></i>
+      	        <span>Investigations</span>
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('investigations_b.add_investigation')}}" id="nav_add_investigation">
+      	        <i class="fas fa-plus"></i>
+      	        <span>Add Investigation</span>
+      	      </a>
+      	    </li>
+          </ul>
+
+
+          <h5 class="d-flex text-muted w-100">
+      		  <span>Objects</span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
+      	        <i class="fas fa-lock-open"></i>
+      	        <span>Decoded</span>
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
+      	        <i class="fas fa-key"></i>
+      	        <span>PGP Dumps</span>
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
+      	        <i class="fas fa-coins"></i>
+      	        <span>Cryptocurrency</span>
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.username_page')}}" id="nav_dashboard_username">
+      	        <i class="fas fa-user"></i>
+      	        <span>Username</span>
+      	      </a>
+      	    </li>
+          </ul>
+
+
+          <h5 class="d-flex text-muted w-100">
+      		  <span>
+              <img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
+               Format
+             </span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">
+                <b>Import</b>
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+              <a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
+                <b>Export</b>
+              </a>
+            </li>
+          </ul>
+      	</nav>
+      </div>
+
+      <script>
+      function toggle_sidebar(){
+      	if($('#nav_menu').is(':visible')){
+      		$('#nav_menu').hide();
+      		$('#side_menu').removeClass('border-right')
+      		$('#side_menu').removeClass('col-lg-2')
+      		$('#core_content').removeClass('col-lg-10')
+      	}else{
+      		$('#nav_menu').show();
+      		$('#side_menu').addClass('border-right')
+      		$('#side_menu').addClass('col-lg-2')
+      		$('#core_content').addClass('col-lg-10')
+      	}
+      }
+      </script>