chg: [MISP export + redis] refactor MISP, export all relationships between objects, relationships between investigated objects + bump redis-py version + theHive export

bin/DB_KVROCKS_MIGRATION.py

@@ -585,6 +585,7 @@ def domain_migration():
 
                     #print()
 
+    # TODO REMOVE INVALID DOMAINS
     for domain_type in ['onion', 'regular']:
         for date in Date.get_date_range_today('20190101'):
             for dom in get_domain_down_by_date(domain_type, date):
@@ -601,7 +602,7 @@ def domain_migration():
                 domain.update_daterange(last_check)
                 if last_origin:
                     domain.set_last_origin(last_origin)
-                domain.add_history(None, None, date=date)
+                domain.add_history(0, None, date=date)
 
 
 ###############################
@@ -871,12 +872,12 @@ if __name__ == '__main__':
     #core_migration()
     #user_migration()
     #tags_migration()
-    items_migration()
+    # items_migration()
-    #crawler_migration()
+    # crawler_migration()
-    # domain_migration()                      # TO TEST ###########################
+    domain_migration()                      # TO TEST ###########################
     # decodeds_migration()
     # screenshots_migration()
-    # subtypes_obj_migration()
+    subtypes_obj_migration()
     # ail_2_ail_migration()
     # trackers_migration()
     # investigations_migration()

bin/export/AILObjects.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import sys
-import uuid
-
-sys.path.append(os.environ['AIL_BIN'])
-from lib.objects import ail_objects
-from lib import ConfigLoader
-
-config_loader = ConfigLoader.ConfigLoader()
-r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
-config_loader = None
-
-def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
-    r_serv_objects.sadd('all_object:uuid', obj_uuid)
-    r_serv_objects.sadd('all_object:global_id', global_id)
-    r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
-    r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
-
-def create_map_obj_event_uuid(event_uuid, global_id):
-    r_serv_objects.sadd('export:all_object:event_uuid', event_uuid)
-    r_serv_objects.sadd('export:all_object:global_id', global_id)
-    r_serv_objects.sadd('object:map:event_id:{}'.format(event_uuid), global_id)
-    r_serv_objects.sadd('object:map:id_event:{}'.format(global_id), event_uuid)
-
-# def get_user_list_of_obj_to_export(user_id, add_uuid=False):
-#     objs_to_export = []
-#     res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
-#     for global_id in res:
-#         dict_obj = Correlate_object.get_global_id_from_id(global_id)
-#         dict_obj['lvl'] = int(res[global_id])
-#         if add_uuid:
-#             obj_dict['uuid'] = str(uuid.uuid4())
-#         objs_to_export.append(dict_obj)
-#     return objs_to_export
-
-def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
-    ## TODO: check if user exist
-    # # TODO: check if valid object
-    # # TODO: check lvl
-    global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
-    return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
-
-def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
-    ## TODO: check if user exist
-    global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
-    r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
-
-def delete_all_user_object_to_export(user_id):
-    ## TODO: check if user exist
-    r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))

bin/export/Export.py

@@ -3,53 +3,172 @@
 
 import os
 import sys
-from uuid import uuid4
+import uuid
 
 sys.path.append(os.environ['AIL_BIN'])
-from lib import ConfigLoader
+##################################
-
+# Import Project packages
-sys.path.append('../../configs/keys')
+##################################
-try:
+from lib.ConfigLoader import ConfigLoader
-    from thehive4py.api import TheHiveApi
+from lib.objects.Items import Item
-    import thehive4py.exceptions
+from lib.ail_core import get_ail_uuid
-    from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
+from lib.Investigations import Investigation
-    if the_hive_url == '':
+from lib.objects import ail_objects
-        is_hive_connected = False
-    else:
-        is_hive_connected = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
-except:
-    is_hive_connected = False
-if is_hive_connected != False:
-    try:
-        is_hive_connected.get_alert(0)
-        is_hive_connected = True
-    except thehive4py.exceptions.AlertException:
-        is_hive_connected = False
 
 ## LOAD CONFIG ##
-config_loader = ConfigLoader.ConfigLoader()
+config_loader = ConfigLoader()
-r_serv_cache = config_loader.get_redis_conn("Redis_Cache")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
-r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+r_db = config_loader.get_db_conn("Kvrocks_DB")
-r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")  ######################################
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")  ######################################
 config_loader = None
 ## -- ##
 
-def load_tags_to_export_in_cache():
+sys.path.append('../../configs/keys')
-    all_exports = ['misp', 'thehive']
+##################################
-    for export_target in all_exports:
+# Import Keys
-        # save solo tags in cache
+##################################
-        all_tags_to_export = Tag.get_list_of_solo_tags_to_export_by_type()
+from thehive4py.api import TheHiveApi
-        if len(all_tags_to_export) > 1:
+from thehive4py.models import Alert, AlertArtifact, Case, CaseObservable
-            r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), *all_tags_to_export)
+import thehive4py.exceptions
-        elif all_tags_to_export:
-            r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), all_tags_to_export[0])
 
-        # save combinaison of tags in cache
+from pymisp import MISPEvent, MISPObject, PyMISP
-        pass
 
-def is_hive_connected(): # # TODO: REFRACTOR, put in cache (with retry)
+##################################
-    return is_hive_connected
+#           THE HIVE
+##################################
 
+HIVE_CLIENT = None
+try:
+    from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
+
+    HIVE_URL = the_hive_url
+    HIVE_KEY = the_hive_key
+    HIVE_VERIFY_CERT = the_hive_verifycert
+except:
+    HIVE_URL = None
+    HIVE_KEY = None
+    HIVE_VERIFY_CERT = None
+
+
+def get_hive_client():
+    global HIVE_CLIENT
+    try:
+        HIVE_CLIENT = TheHiveApi(HIVE_URL, HIVE_KEY, cert=HIVE_VERIFY_CERT)
+    except:
+        HIVE_CLIENT = None
+    return HIVE_CLIENT
+
+
+def is_hive_connected():
+    try:
+        # print(hive_client.health())
+        HIVE_CLIENT.get_alert(0)
+        return True
+    except thehive4py.exceptions.AlertException:
+        return False
+
+
+HIVE_CLIENT = get_hive_client()
+
+
+def create_thehive_alert(item_id, tag_trigger):
+    item = Item(item_id)
+    meta = item.get_meta()
+    # TheHive expects a file
+    content = item.get_raw_content(decompress=True)
+
+    # remove .gz from submitted path to TheHive because we've decompressed it
+    if item_id.endswith(".gz"):
+        item_id = item_id[:-3]
+    # add .txt it's easier to open when downloaded from TheHive
+    item_id = f'{item_id}.txt'
+
+    artifacts = [
+        AlertArtifact(dataType='other', message='uuid-ail', data=(get_ail_uuid())),
+        AlertArtifact(dataType='file', data=(content, item_id), tags=meta['tags'])
+    ]
+
+    # Prepare the sample Alert
+    sourceRef = str(uuid.uuid4())[0:6]
+    alert = Alert(title='AIL Leak',
+                  tlp=3,
+                  tags=meta['tags'],
+                  description='AIL Leak, triggered by {}'.format(tag_trigger),
+                  type='ail',
+                  source=meta['source'],  # Use item ID ?
+                  sourceRef=sourceRef,
+                  artifacts=artifacts)
+
+    # Create the Alert
+    alert_id = None
+    try:
+        response = HIVE_CLIENT.create_alert(alert)
+        if response.status_code == 201:
+            # print(json.dumps(response.json(), indent=4, sort_keys=True))
+            print('Alert Created')
+            print(response.json())
+            alert_id = response.json()['id']
+        else:
+            print(f'ko: {response.status_code}/{response.text}')
+            return 0
+    except:
+        print('hive connection error')
+    print(alert_id)
+
+
+# TODO SAVE CASE URL ????????????????????????
+def create_thehive_case(item_id, title=None, tlp=2, threat_level=2, description=None):
+    item = Item(item_id)
+    ail_uuid = get_ail_uuid()
+
+    if not title:
+        title = f'AIL Case {item.id}'
+    if not description:
+        description = f'AIL {ail_uuid} Case'
+    date = item.get_date()
+    date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}'
+    tags = item.get_tags(r_list=True)
+
+    case = Case(title=title,
+                tlp=tlp,
+                severity=threat_level,
+                flag=False,
+                tags=tags,
+                description=description)
+
+    # Create Case
+    response = get_hive_client().create_case(case)
+    if response.status_code == 201:
+        case_id = response.json()['id']
+
+        observables = [
+            CaseObservable(dataType="other", data=[ail_uuid], message="uuid-ail"),
+            CaseObservable(dataType="file", data=item.get_filename(), tags=tags),
+            CaseObservable(dataType="other", data=[item.get_source()], message="source"),
+            CaseObservable(dataType="other", data=[date], message="last-seen")
+        ]
+
+        for observable in observables:
+            resp = HIVE_CLIENT.create_case_observable(case_id, observable)
+            if resp.status_code != 201:
+                print(f'error observable creation: {resp.status_code}/{resp.text}')
+        # print(case_id)
+        # return HIVE_URL /thehive/cases/~37040/details
+        return case_id
+
+        # r_serv_metadata.set('hive_cases:'+path, id)
+    else:
+        print(f'ko: {response.status_code}/{response.text}')
+        return None
+
+
+def get_case_url(case_id):
+    return f'{HIVE_URL}/cases/{case_id}/details'
+
+
+# TODO
 def get_item_hive_cases(item_id):
     hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id))
     if hive_case:
@@ -57,6 +176,150 @@ def get_item_hive_cases(item_id):
     return hive_case
 
 
+##################################
+#           MISP
+##################################
+
+try:
+    from mispKEYS import misp_url, misp_key, misp_verifycert
+
+    MISP_URL = misp_url
+    MISP_KEY = misp_key
+    MISP_VERIFYCERT = misp_verifycert
+    if MISP_URL.endswith('/'):
+        MISP_URL = MISP_URL[:-1]
+except:
+    MISP_URL = None
+    MISP_KEY = None
+    MISP_VERIFYCERT = None
+
+
+def get_misp_client():
+    return PyMISP(misp_url, misp_key, misp_verifycert)
+
+
+# # TODO: return error
+def ping_misp():
+    try:
+        PyMISP(misp_url, misp_key, misp_verifycert)
+        return True
+    except Exception as e:
+        print(e)
+        return False
+
+
+def sanitize_misp_event_distribution(distribution):
+    try:
+        int(distribution)
+        if 0 <= distribution <= 3:
+            return distribution
+        else:
+            return 0
+    except:
+        return 0
+
+
+def sanitize_misp_event_threat_level(threat_level):
+    try:
+        int(threat_level)
+        if 1 <= threat_level <= 4:
+            return threat_level
+        else:
+            return 4
+    except:
+        return 4
+
+
+def sanitize_misp_event_analysis(analysis):
+    try:
+        int(analysis)
+        if 0 <= analysis <= 2:
+            return analysis
+        else:
+            return 0
+    except:
+        return 0
+
+def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0,
+                      info=None, tags=None, export=False):
+    if tags is None:
+        tags = []
+    event = MISPEvent()
+    if not event_uuid:
+        event_uuid = str(uuid.uuid4())
+    event.uuid = event_uuid
+    if date:
+        event.date = date
+    if not info:
+        info = 'AIL framework export'
+    event.info = info
+    if publish:
+        event.publish()
+    for tag in tags:
+        event.add_tag(tag)
+    event.distribution = sanitize_misp_event_distribution(distribution)
+    event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id)
+    event.analysis = sanitize_misp_event_analysis(analysis)
+
+    misp_objects = ail_objects.get_misp_objects(objs)
+    for obj in misp_objects:
+        event.add_object(obj)
+
+    # print(event.to_json())
+
+    if export:
+        misp = get_misp_client()
+        misp_event = misp.add_event(event)
+        # TODO: handle error
+
+        misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}'
+        return misp_event
+    else:
+        return {'uuid': event['uuid'], 'event': event.to_json()}
+
+def create_investigation_misp_event(investigation_uuid):
+    investigation = Investigation(investigation_uuid)
+    objs = ail_objects.get_objects(investigation.get_objects())
+
+    event = create_misp_event(objs,
+                              # event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ????
+                              date=investigation.get_date(),
+                              distribution=0,
+                              threat_level_id=investigation.get_threat_level(),
+                              analysis=investigation.get_analysis(),
+                              info=investigation.get_info(),
+                              tags=investigation.get_tags(),
+                              export=True)
+    url = event['url']
+    if url:
+        investigation.add_misp_events(url)
+    return url
+
+def get_user_misp_objects_to_export(user_id):
+    objs = []
+    objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
+    for obj in objects:
+        obj_type, obj_subtype, obj_id = obj.split(':', 2)
+        lvl = objects[obj]
+        try:
+            lvl = int(lvl)
+        except:
+            lvl = 0
+        objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
+    return objs
+
+def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
+    if not obj_subtype:
+        obj_subtype = ''
+    r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
+
+def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
+    r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
+
+def delete_user_misp_objects_to_export(user_id):
+    r_db.delete(f'user:obj:misp:export:{user_id}')
+
+
 ###########################################################
 # # set default
 # if r_serv_db.get('hive:auto-alerts') is None:
@@ -64,3 +327,11 @@ def get_item_hive_cases(item_id):
 #
 # if r_serv_db.get('misp:auto-events') is None:
 #     r_serv_db.set('misp:auto-events', 0)
+
+# if __name__ == '__main__':
+# from lib.objects.Cves import Cve
+# create_misp_event([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
+#                       Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
+#                       Cve('CVE-2015-0410')])
+
+# create_investigation_misp_event('c6bbf8fa9ead4cc698eaeb07835cca5d)

bin/export/MispExport.py

@@ -1,329 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import io
-import sys
-import uuid
-
-sys.path.append(os.environ['AIL_BIN'])
-from lib.objects import ail_objects
-
-from export import AILObjects
-
-
-from lib.Investigations import Investigation
-
-# # TODO: # FIXME: REFACTOR ME => use UI/Global config
-sys.path.append('../../configs/keys')
-try:
-    from mispKEYS import misp_url, misp_key, misp_verifycert
-except:
-    misp_url = ''
-    misp_key = ''
-    misp_verifycert = False
-
-# MISP
-from pymisp import MISPEvent, MISPObject, PyMISP
-
-def is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
-    if not ail_objects.is_valid_object_type(obj_type):
-        return False
-    if not ail_objects.is_valid_object_subtype(obj_type, obj_subtype):
-        return False
-    if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
-        return False
-    return True
-
-def sanitize_obj_export_lvl(lvl):
-    try:
-        lvl = int(lvl)
-    except:
-        lvl = 0
-    return lvl
-
-def get_export_filename(json_content):
-    return 'ail_export_{}.json'.format(json_content.uuid)
-
-def create_in_memory_file(json_content):
-    return io.BytesIO(json_content.encode())
-
-def add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj):
-    global_id = ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
-    global_id_new = ail_objects.get_obj_global_id(dict_new_obj['type'], dict_new_obj.get('subtype', ''), dict_new_obj['id'])
-    if global_id > global_id_new:
-        res = (global_id, global_id_new)
-    else:
-        res = (global_id_new, global_id)
-    set_relationship.add( res )
-
-# # TODO: add action by obj type
-# ex => Domain
-def add_obj_to_create(all_obj_to_export, set_relationship, dict_obj):
-    all_obj_to_export.add(ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id']))
-
-def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl):
-    # # TODO: filter by export mode or filter on all global ?
-    if lvl >= 0:
-        add_obj_to_create(all_obj_to_export, add_obj_to_create, dict_obj)
-
-    if lvl > 0:
-        lvl = lvl - 1
-
-        # # TODO: filter by correlation types
-        obj_correlations = ail_objects.get_obj_correlations(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
-        for obj_type in obj_correlations:
-            dict_new_obj = {'type': obj_type}
-            if obj_type=='pgp' or obj_type=='cryptocurrency' or obj_type=='username':
-                for subtype in obj_correlations[obj_type]:
-                    dict_new_obj['subtype'] = subtype
-                    for obj_id in obj_correlations[obj_type][subtype]:
-                        dict_new_obj['id'] = obj_id
-                        add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
-                        add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
-
-            else:
-                for obj_id in obj_correlations[obj_type]:
-                    dict_new_obj['id'] = obj_id
-                    add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
-                    add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
-
-
-        add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
-
-
-def create_list_of_objs_to_export(l_obj, r_type='json'):
-    all_obj_to_export = set()
-    set_relationship = set()
-    for obj in l_obj:
-        add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, obj, obj.get('lvl', 1))
-
-    # create MISP objects
-    dict_misp_obj = create_all_misp_obj(all_obj_to_export, set_relationship)
-
-    # create object relationships
-    for obj_global_id_1, obj_global_id_2 in set_relationship:
-        dict_relationship = get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2)
-        if dict_relationship:
-            obj_src = dict_misp_obj[dict_relationship['src']]
-            obj_dest = dict_misp_obj[dict_relationship['dest']]
-            obj_src.add_reference(obj_dest.uuid, dict_relationship['relation'], 'add a comment')
-
-    event = MISPEvent()
-    event.info = 'AIL framework export'
-    event.uuid = str(uuid.uuid4())
-    for obj_global_id in dict_misp_obj:
-        misp_obj = dict_misp_obj[obj_global_id]
-        AILObjects.create_map_obj_event_uuid(event.uuid, obj_global_id)
-        AILObjects.create_map_obj_uuid_golbal_id(event.uuid, obj_global_id)
-        if misp_obj:
-            # add object to event
-            event.add_object(dict_misp_obj[obj_global_id])
-
-    return event
-
-# TODO REFACTOR ME
-def create_all_misp_obj(all_obj_to_export, set_relationship):
-    dict_misp_obj = {}
-    for obj_global_id in all_obj_to_export:
-        obj_type, obj_id = obj_global_id.split(':', 1)
-        dict_misp_obj[obj_global_id] = create_misp_obj(obj_type, obj_id)
-    return dict_misp_obj
-
-# TODO REFACTOR ME
-def create_misp_obj(obj_type, obj_id):
-    if obj_type in ['cryptocurrency', 'pgp', 'username']:
-        obj_subtype, obj_id = obj_id.split(':', 1)
-    else:
-        obj_subtype = ''
-    misp_obj = ail_objects.get_misp_object(obj_type, obj_subtype, obj_id)
-    return misp_obj
-
-
-def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
-    obj_type_1 = obj_global_id_1.split(':', 1)[0]
-    obj_type_2 = obj_global_id_2.split(':', 1)[0]
-    type_tuple = [obj_type_1, obj_type_2]
-
-    if 'image' in type_tuple: # or screenshot ## TODO:
-        if obj_type_1 == 'image':
-            src = obj_global_id_1
-            dest = obj_global_id_2
-        else:
-            src = obj_global_id_2
-            dest = obj_global_id_1
-        return {'relation': 'screenshot-of', 'src': src, 'dest': dest}
-    elif 'decoded' in type_tuple:
-        if obj_type_1 == 'decoded':
-            src = obj_global_id_1
-            dest = obj_global_id_2
-        else:
-            src = obj_global_id_2
-            dest = obj_global_id_1
-        return {'relation': 'included-in', 'src': src, 'dest': dest}
-    elif 'pgp' in type_tuple:
-        if obj_type_1 == 'pgp':
-            src = obj_global_id_1
-            dest = obj_global_id_2
-        else:
-            src = obj_global_id_2
-            dest = obj_global_id_1
-        return {'relation': 'extracted-from', 'src': src, 'dest': dest}
-    elif 'cryptocurrency':
-        if obj_type_1 == 'cryptocurrency':
-            src = obj_global_id_1
-            dest = obj_global_id_2
-        else:
-            src = obj_global_id_2
-            dest = obj_global_id_1
-        return {'relation': 'extracted-from', 'src': src, 'dest': dest}
-    elif 'domain' in type_tuple:
-        if 'item' in type_tuple:
-            if obj_type_1 == 'item':
-                src = obj_global_id_1
-                dest = obj_global_id_2
-            else:
-                src = obj_global_id_2
-                dest = obj_global_id_1
-            return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
-    elif 'item' in type_tuple:
-        if 'domain' in type_tuple:
-            if obj_type_1 == 'item':
-                src = obj_global_id_1
-                dest = obj_global_id_2
-            else:
-                src = obj_global_id_2
-                dest = obj_global_id_1
-            return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
-    return None
-
-def sanitize_event_distribution(distribution):
-    try:
-        int(distribution)
-        if (0 <= distribution <= 3):
-            return distribution
-        else:
-            return 0
-    except:
-        return 0
-
-def sanitize_event_threat_level_id(threat_level_id):
-    try:
-        int(threat_level_id)
-        if (1 <= threat_level_id <= 4):
-            return threat_level_id
-        else:
-            return 4
-    except:
-        return 4
-
-def sanitize_event_analysis(analysis):
-    try:
-        int(analysis)
-        if (0 <= analysis <= 2):
-            return analysis
-        else:
-            return 0
-    except:
-        return 0
-
-# # TODO: return error
-def ping_misp():
-    try:
-        PyMISP(misp_url, misp_key, misp_verifycert)
-        return True
-    except Exception as e:
-        print(e)
-        return False
-
-def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
-    if event_info:
-        event.info = event_info
-    event.distribution = sanitize_event_distribution(distribution)
-    event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
-    event.analysis = sanitize_event_analysis(analysis)
-    if publish:
-        event.publish()
-
-    # # TODO: handle multiple MISP instance
-    misp = PyMISP(misp_url, misp_key, misp_verifycert)
-    #print(event.to_json())
-
-    misp_event = misp.add_event(event)
-     #print(misp_event)
-    # # TODO: handle error
-    event_metadata = extract_event_metadata(misp_event)
-    return event_metadata
-
-def extract_event_metadata(event):
-    event_metadata = {}
-    event_metadata['uuid'] = event['Event']['uuid']
-    event_metadata['id'] = event['Event']['id']
-    if misp_url[-1] == '/':
-        event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id'])
-    else:
-        event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id'])
-    return event_metadata
-
-######
-#
-# EXPORT LVL DEFINITION: (== Correl<tion DEPTH)
-#
-# LVL 0 => PARTIAL    Only add core item Correlation
-# LVL 1 => DETAILED   Also add correlated_items correlation
-######
-
-# # TODO: # create object relationships
-def create_investigation_event(investigation_uuid):
-    investigation = Investigation(investigation_uuid)
-
-    event = MISPEvent()
-    event.info = investigation.get_info()
-    event.uuid = investigation.get_uuid(separator=True)
-    event.date = investigation.get_date()
-    event.analysis = investigation.get_analysis()
-    event.threat_level_id = investigation.get_threat_level()
-
-    event.distribution = 0
-
-    # tags
-    for tag in investigation.get_tags():
-        event.add_tag(tag)
-    # objects
-    investigation_objs = investigation.get_objects()
-    for obj in investigation_objs:
-        # if subtype -> obj_id = 'subtype:type'
-        if obj['subtype']:
-            obj_id = f"{obj['subtype']}:{obj['id']}"
-        else:
-            obj_id = obj['id']
-        misp_obj = create_misp_obj(obj['type'], obj_id)
-        if misp_obj:
-            event.add_object(misp_obj)
-
-    # if publish:
-    #     event.publish()
-
-    # print(event.to_json())
-    misp = PyMISP(misp_url, misp_key, misp_verifycert)
-    if misp.event_exists(event.uuid):
-        misp_event = misp.update_event(event)
-    else:
-        misp_event = misp.add_event(event)
-
-    # # TODO: handle error
-    event_metadata = extract_event_metadata(misp_event)
-    if event_metadata.get('uuid'):
-        if misp_url[-1] == '/':
-            url =  misp_url[:-1]
-        else:
-            url =  misp_url
-        investigation.add_misp_events(url)
-    return event_metadata
-
-# if __name__ == '__main__':
-
-    # l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
-    # create_list_of_objs_to_export(l_obj)
-
-    #print(event.to_json())

bin/export/MispImport.py

@@ -13,8 +13,6 @@ from lib.objects import ail_objects
 
 from lib.objects import Items
 
-from export import AILObjects
-
 # MISP
 from pymisp import MISPEvent, MISPObject, PyMISP
 
@@ -218,10 +216,6 @@ def create_obj_relationships(map_uuid_global_id, misp_obj):
 
                 # TODO CREATE OBJ RELATIONSHIP
 
-def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
-    for obj_uuid in map_uuid_global_id:
-        AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
-
 def import_objs_from_file(filepath):
     map_uuid_global_id = {}
 
@@ -237,7 +231,6 @@ def import_objs_from_file(filepath):
     for misp_obj in event_to_import.objects:
         create_obj_relationships(map_uuid_global_id, misp_obj)
 
-    create_map_all_obj_uuid_golbal_id(map_uuid_global_id)
     return map_uuid_global_id
 
 

bin/import/ail_json_importer/Ail_feeder_urlextract.py

@@ -12,12 +12,16 @@ import sys
 import datetime
 import uuid
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
-import item_basic
-
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
 from Default_json import Default_json
 
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects.Items import Item
+
+
 class Ail_feeder_urlextract(Default_json):
     """urlextract Feeder functions"""
 
@@ -46,13 +50,14 @@ class Ail_feeder_urlextract(Default_json):
 
     # # TODO:
     def process_json_meta(self, process, item_id):
-        '''
+        """
         Process JSON meta filed.
-        '''
+        """
         json_meta = self.get_json_meta()
+        parent_id = str(json_meta['parent:twitter:tweet_id']) # TODO SEARCH IN CACHE !!!
+        item = Item(item_id)
+        item.set_parent(parent_id)
 
         # # TODO: change me
-        parent_type = 'twitter_id'
+        # parent_type = 'twitter_id'
-
+        # item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)
-        parent_id = str(json_meta['parent:twitter:tweet_id'])
-        item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)

bin/lib/Investigations.py

@@ -200,8 +200,8 @@ class Investigation(object):
         else:
             raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
 
-    def add_misp_events(self, misp_url):
+    def add_misp_events(self, event_uuid):
-        r_tracking.sadd(f'investigations:misp:{self.uuid}', misp_url)
+        r_tracking.sadd(f'investigations:misp:{self.uuid}', event_uuid)
 
     def set_tags(self, tags):
         # delete previous tags

bin/lib/ail_core.py

@@ -12,13 +12,13 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib.ConfigLoader import ConfigLoader
 
 config_loader = ConfigLoader()
-
+r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
 config_loader = None
 
-AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}
+AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'})
 
 def get_ail_uuid():
-    pass
+    return r_serv_db.get('ail:uuid')
 
 #### AIL OBJECTS ####
 
@@ -33,11 +33,21 @@ def get_object_all_subtypes(obj_type):
     if obj_type == 'username':
         return ['telegram', 'twitter', 'jabber']
 
+def get_all_objects_with_subtypes_tuple():
+    str_objs = []
+    for obj_type in get_all_objects():
+        subtypes = get_object_all_subtypes(obj_type)
+        if subtypes:
+            for subtype in subtypes:
+                str_objs.append((obj_type, subtype))
+        else:
+            str_objs.append((obj_type, ''))
+    return str_objs
+
 ##-- AIL OBJECTS --##
 
 def paginate_iterator(iter_elems, nb_obj=50, page=1):
-    dict_page = {}
+    dict_page = {'nb_all_elem': len(iter_elems)}
-    dict_page['nb_all_elem'] = len(iter_elems)
     nb_pages = dict_page['nb_all_elem'] / nb_obj
     if not nb_pages.is_integer():
         nb_pages = int(nb_pages)+1

bin/lib/data_retention_engine.py

@@ -96,4 +96,3 @@ def update_obj_date(date, obj_type, subtype=''):
 if __name__ == '__main__':
     print(r_obj.hgetall(f'date:first'))
     print(r_obj.hgetall(f'date:last'))
-

bin/lib/objects/Domains.py

@@ -420,6 +420,10 @@ class Domain(AbstractObject):
     def add_history(self, epoch, root_item=None, date=None):
         if not date:
             date = time.strftime('%Y%m%d', time.gmtime(epoch))
+        if not root_item:
+            root_item = int(epoch)
+            status = False
+        else:
             try:
                 root_item = int(root_item)
                 status = False

bin/lib/objects/Items.py

@@ -88,8 +88,11 @@ class Item(AbstractObject):
         else:
             return item_basic.get_item_content(self.id)
 
-    def get_raw_content(self):
+    def get_raw_content(self, decompress=False):
         filepath = self.get_filename()
+        if decompress:
+            raw_content = BytesIO(self.get_content(binary=True))
+        else:
             with open(filepath, 'rb') as f:
                 raw_content = BytesIO(f.read())
         return raw_content

bin/lib/objects/Usernames.py

@@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
 from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 
 config_loader = ConfigLoader()
-
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None
 
 

bin/lib/objects/abstract_object.py

@@ -243,6 +243,14 @@ class AbstractObject(ABC):
         """
         return is_obj_correlated(self.type, self.subtype, self.id, type2, subtype2, id2)
 
+    def are_correlated(self, object2):
+        """
+        Check if an object is correlated with another Object
+        :type object2 AbstractObject
+        """
+        return is_obj_correlated(self.type, self.subtype, self.id,
+                                 object2.get_type(), object2.get_subtype(r_str=True), object2.get_id())
+
     def delete_correlation(self, type2, subtype2, id2):
         """
         Get object correlations

bin/lib/objects/ail_objects.py

@@ -4,15 +4,12 @@
 import os
 import sys
 
-from abc import ABC
-from flask import url_for
-
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.ail_core import get_all_objects
+from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple
 from lib import correlations_engine
 from lib import btc_ail
 from lib import Tag
@@ -26,26 +23,18 @@ from lib.objects import Pgps
 from lib.objects.Screenshots import Screenshot
 from lib.objects import Usernames
 
-
 config_loader = ConfigLoader()
 
 config_loader = None
 
-class AILObjects(object):  ## ??????????????????????
+
-    initial = 0
+
-    ongoing = 1
-    completed = 2
 
 def is_valid_object_type(obj_type):
     return obj_type in get_all_objects()
 
 def is_valid_object_subtype(obj_type, subtype):
-    if obj_type == 'cryptocurrency':
+    return subtype in get_object_all_subtypes(obj_type)
-        return subtype in CryptoCurrencies.get_all_subtypes()
-    elif obj_type == 'pgp':
-        return subtype in Pgps.get_all_subtypes()
-    elif obj_type == 'username':
-        return subtype in CryptoCurrencies.get_all_subtypes()
 
 def sanitize_objs_types(objs):
     l_types = []
@@ -56,6 +45,7 @@ def sanitize_objs_types(objs):
         l_types = get_all_objects()
     return l_types
 
+
 def get_object(obj_type, subtype, id):
     if obj_type == 'item':
         return Item(id)
@@ -74,6 +64,25 @@ def get_object(obj_type, subtype, id):
     elif obj_type == 'username':
         return Usernames.Username(id, subtype)
 
+def get_objects(objects):
+    objs = set()
+    for obj in objects:
+        if isinstance(obj, dict):
+            obj_type = obj['type']
+            obj_subtype = obj['subtype']
+            obj_id = obj['id']
+            if 'lvl' in obj:
+                correl_objs = get_obj_correlations_objs(obj_type, obj_subtype, obj_id, lvl=obj['lvl'])
+                objs = objs.union(correl_objs)
+        else:
+            obj_type, obj_subtype, obj_id = obj
+        objs.add((obj_type, obj_subtype, obj_id))
+    ail_objects = []
+    for obj in objs:
+        ail_objects.append(get_object(obj[0], obj[1], obj[2]))
+    return ail_objects
+
+
 def exists_obj(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
     if obj:
@@ -81,35 +90,44 @@ def exists_obj(obj_type, subtype, obj_id):
     else:
         return False
 
+
 def get_obj_global_id(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.get_global_id()
 
+
 def get_obj_from_global_id(global_id):
     obj = global_id.split(':', 3)
     return get_object(obj[0], obj[1], obj[2])
 
+
 def get_object_link(obj_type, subtype, id, flask_context=False):
     obj = get_object(obj_type, subtype, id)
     return obj.get_link(flask_context=flask_context)
 
+
 def get_object_svg(obj_type, subtype, id):
     obj = get_object(obj_type, subtype, id)
     return obj.get_svg_icon()
 
+
 ## TAGS ##
 def get_obj_tags(obj_type, subtype, id):
     obj = get_object(obj_type, subtype, id)
     return obj.get_tags()
 
+
 def add_obj_tag(obj_type, subtype, id, tag):
     obj = get_object(obj_type, subtype, id)
     obj.add_tag(tag)
 
+
 def add_obj_tags(obj_type, subtype, id, tags):
     obj = get_object(obj_type, subtype, id)
     for tag in tags:
         obj.add_tag(tag)
+
+
 # -TAGS- #
 
 def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
@@ -119,6 +137,7 @@ def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
     meta['link'] = obj.get_link(flask_context=flask_context)
     return meta
 
+
 def get_objects_meta(objs, options=[], flask_context=False):
     metas = []
     for obj_dict in objs:
@@ -126,12 +145,14 @@ def get_objects_meta(objs, options=[], flask_context=False):
                                      flask_context=flask_context))
     return metas
 
+
 def get_object_card_meta(obj_type, subtype, id, related_btc=False):
     obj = get_object(obj_type, subtype, id)
     meta = obj.get_meta()
     meta['icon'] = obj.get_svg_icon()
     if subtype or obj_type == 'cve':
         meta['sparkline'] = obj.get_sparkline()
+        if obj_type == 'cve':
             meta['cve_search'] = obj.get_cve_search()
     if subtype == 'bitcoin' and related_btc:
         meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id)
@@ -143,6 +164,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
         meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
     return meta
 
+
 def get_ui_obj_tag_table_keys(obj_type):
     '''
     Warning: use only in flask (dynamic templates)
@@ -150,15 +172,58 @@ def get_ui_obj_tag_table_keys(obj_type):
     if obj_type == "domain":
         return ['id', 'first_seen', 'last_check', 'status']  # # TODO: add root screenshot
 
+
+# # # # MISP OBJECTS # # # #
+
 # # TODO: CHECK IF object already have an UUID
 def get_misp_object(obj_type, subtype, id):
     obj = get_object(obj_type, subtype, id)
     return obj.get_misp_object()
 
+
+def get_misp_objects(objs):
+    misp_objects = {}
+    for obj in objs:
+        misp_objects[obj] = obj.get_misp_object()
+    for relation in get_objects_relationships(objs):
+        obj_src = misp_objects[relation['src']]
+        obj_dest = misp_objects[relation['dest']]
+        # print(relation['src'].get_id(), relation['dest'].get_id())
+        obj_src.add_reference(obj_dest.uuid, relation['relationship'], 'ail correlation')
+    return misp_objects.values()
+
 # get misp relationship
-def get_objects_relationship(obj_1, obj2):
+def get_objects_relationships(objs):
-    relationship = {}
+    relation = []
-    obj_types = (obj_1.get_type(), obj2.get_type())
+    if len(objs) == 2:
+        if objs[0].are_correlated(objs[1]):
+            relationship = get_objects_relationship(objs[0], objs[1])
+            if relationship:
+                relation.append(relationship)
+    else:
+        iterator = objs.copy()  # [obj1, obj2, obj3, obj4]
+        for obj in objs[:-1]:  # [obj1, obj2, obj3]
+            iterator.pop(0)  # [obj2, obj3, obj4] obj1 correlation already checked
+            for obj2 in iterator:
+                # CHECK CORRELATION obj - > obj2
+                if obj.are_correlated(obj2):
+                    relationship = get_objects_relationship(obj, obj2)
+                    if relationship:
+                        relation.append(relationship)
+    return relation
+
+def get_relationship_src_dest(src_type, obj1, obj2):
+    if obj1.get_type() == src_type:
+        src = obj1
+        dest = obj2
+    else:
+        src = obj2
+        dest = obj1
+    return src, dest
+
+# get misp relationship
+def get_objects_relationship(obj1, obj2):
+    obj_types = (obj1.get_type(), obj2.get_type())
 
     ##############################################################
     # if ['cryptocurrency', 'pgp', 'username', 'decoded', 'screenshot']:
@@ -166,63 +231,79 @@ def get_objects_relationship(obj_1, obj2):
     #     relationship[relation] =
     ##############################################################
     if 'cryptocurrency' in obj_types:
-        relationship['relation'] = 'extracted-from'
+        relationship = 'extracted-from'
-        if obj_1.get_type() == 'cryptocurrency':
+        src, dest = get_relationship_src_dest('cryptocurrency', obj1, obj2)
-            relationship['src'] = obj_1.get_id()
+    elif 'cve' in obj_types:
-            relationship['dest'] = obj2.get_id()
+        relationship = 'extracted-from'
-        else:
+        src, dest = get_relationship_src_dest('cve', obj1, obj2)
-            relationship['src'] = obj2.get_id()
-            relationship['dest'] = obj_1.get_id()
-
     elif 'pgp' in obj_types:
-        relationship['relation'] = 'extracted-from'
+        relationship = 'extracted-from'
-
+        src, dest = get_relationship_src_dest('pgp', obj1, obj2)
     elif 'username' in obj_types:
-        relationship['relation'] = 'extracted-from'
+        relationship = 'extracted-from'
-
+        src, dest = get_relationship_src_dest('username', obj1, obj2)
     elif 'decoded' in obj_types:
-        relationship['relation'] = 'included-in'
+        relationship = 'included-in'
-
+        src, dest = get_relationship_src_dest('decoded', obj1, obj2)
     elif 'screenshot' in obj_types:
-        relationship['relation'] = 'screenshot-of'
+        relationship = 'screenshot-of'
-
+        src, dest = get_relationship_src_dest('screenshot', obj1, obj2)
     elif 'domain' in obj_types:
-        relationship['relation'] = 'extracted-from'
+        relationship = 'extracted-from'
-
+        src, dest = get_relationship_src_dest('domain', obj1, obj2)
     # default
     else:
-        pass
+        relationship = None
+        src = None
+        dest = None
+    if not relationship:
+        return {}
+    else:
+        return {'src': src, 'dest': dest, 'relationship': relationship}
+
+# - - - MISP OBJECTS - - - #
 
-    return relationship
 
 def api_sanitize_object_type(obj_type):
     if not is_valid_object_type(obj_type):
         return {'status': 'error', 'reason': 'Incorrect object type'}, 400
 
-def get_obj_correlations(obj_type, subtype, id):
+
-    obj = get_object(obj_type, subtype, id)
+def get_obj_correlations(obj_type, subtype, obj_id):
+    obj = get_object(obj_type, subtype, obj_id)
     return obj.get_correlations()
 
-################################################################################
+def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
-# DATA RETENTION
+    if lvl > 0 and (obj_type, subtype, obj_id) not in objs:  # Avoid looking for the same correlation
-# # TODO: TO ADD ??????????????????????
+        objs.add((obj_type, subtype, obj_id))
-# def get_first_objects_date():
+        lvl = lvl - 1
-#     return r_object.zrange('objs:first_date', 0, -1)
+        obj = get_object(obj_type, subtype, obj_id)
-#
+        correlations = obj.get_correlations()
-# def get_first_object_date(obj_type, subtype):
+        # print('--------------------------')
-#     return r_object.zscore('objs:first_date', f'{obj_type}:{subtype}')
+        # print( obj_id, correlations)
-#
+        # print(lvl)
-# def set_first_object_date(obj_type, subtype, date):
+        # print('--------------------------')
-#     return r_object.zadd('objs:first_date', f'{obj_type}:{subtype}', date)
+        for obj2_type in correlations:
+            for str_obj in correlations[obj2_type]:
+                obj2_subtype, obj2_id = str_obj.split(':', 1)
+                _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
+    # print(len(objs))
+    objs.add((obj_type, subtype, obj_id))
+    return objs
 
+def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
+    objs = set()
+    _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
+    return objs
 
 ################################################################################
 ################################################################################
 ################################################################################
 
-def delete_obj(obj_type, subtype, id):
+def delete_obj(obj_type, subtype, obj_id):
-    object = get_object(obj_type, subtype, id)
+    obj = get_object(obj_type, subtype, obj_id)
-    return object.delete()
+    return obj.delete()
+
 
 ################################################################################
 ################################################################################
@@ -236,11 +317,12 @@ def create_correlation_graph_links(links_set):
         links.append({"source": link[0], "target": link[1]})
     return links
 
+
 def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
     graph_nodes_list = []
     for node_id in nodes_set:
         obj_type, subtype, obj_id = node_id.split(';', 2)
-        dict_node = {"id": node_id}
+        dict_node = {'id': node_id}
         dict_node['style'] = get_object_svg(obj_type, subtype, obj_id)
 
         # # TODO: # FIXME: in UI
@@ -258,11 +340,27 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
         graph_nodes_list.append(dict_node)
     return graph_nodes_list
 
-def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
-    obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=flask_context)
-    return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context), "links": create_correlation_graph_links(links)}
-
 
+def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+                                flask_context=False):
+    obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
+                                                                                      filter_types=filter_types,
+                                                                                      max_nodes=max_nodes, level=level,
+                                                                                      flask_context=flask_context)
+    return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context),
+            "links": create_correlation_graph_links(links)}
 
 
 ###############
+
+
+# if __name__ == '__main__':
+    # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
+    # r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
+    #                       Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
+    #                       Cve('CVE-2015-0410')])
+    # print()
+    # print(r)
+
+    # res = get_obj_correlations_objs('username', 'telegram', 'corona', lvl=100)
+    # print(res)

requirements.txt

@@ -6,7 +6,7 @@ d4-pyclient>=0.1.6
 thehive4py
 
 # Core
-redis==3.0.0
+redis>4.4.0
 python-magic>0.4.15
 yara-python>4.0.2
 

var/www/blueprints/crawler_splash.py

@@ -34,7 +34,7 @@ from lib import Tag
 
 from packages import Date
 
-#import Config_DB
+# import Config_DB
 bootstrap_label = Flask_config.bootstrap_label
 
 # ============ BLUEPRINT ============

var/www/blueprints/import_export.py

@@ -4,17 +4,16 @@
 '''
     Blueprint Flask: MISP format import export
 '''
-
+import io
 import os
 import sys
 import uuid
 import json
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file
+from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort
 from flask_login import login_required, current_user, login_user, logout_user
 
 sys.path.append('modules')
-import Flask_config
 
 # Import Role_Manager
 from Role_Manager import login_admin, login_analyst, login_read_only
@@ -23,20 +22,21 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from export import MispExport
+from export import Export
-from export import MispImport
-from export import AILObjects ####################### # # # # ########################## ## # ####################################
-
 from lib.objects import ail_objects
 
+from export import MispImport  # TODO REMOVE ME
+
+# TODO REMOVE ME
 
 # ============ BLUEPRINT ============
-import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
+import_export = Blueprint('import_export', __name__,
+                          template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
+
 
 # ============ VARIABLES ============
 
 
-
 # ============ FUNCTIONS ============
 
 
@@ -47,6 +47,7 @@ import_export = Blueprint('import_export', __name__, template_folder=os.path.joi
 def import_object():
     return render_template("import_object.html")
 
+# TODO
 @import_export.route("/import_export/import_file", methods=['POST'])
 @login_required
 @login_analyst
@@ -79,137 +80,129 @@ def import_object_file():
 
     return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
 
-@import_export.route('/import_export/export')
+
+@import_export.route("/objects/misp/export", methods=['GET'])
 @login_required
 @login_analyst
-def export_object():
+def objects_misp_export():
     user_id = current_user.get_id()
+    object_types = ail_objects.get_all_objects_with_subtypes_tuple()
+    to_export = Export.get_user_misp_objects_to_export(user_id)
+    return render_template("export_object.html", object_types=object_types, to_export=to_export)
 
-    # get user saved obj to export
-    l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
 
-    return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
+@import_export.route("/objects/misp/export/post", methods=['POST'])
-
-@import_export.route("/import_export/export_file", methods=['POST'])
 @login_required
 @login_analyst
-def export_object_file():
+def objects_misp_export_post():
     user_id = current_user.get_id()
 
-    l_obj_to_export = []
-    l_obj_invalid = []
-
-    export_to_misp = False
-    dict_misp_event_export = {}
-
     # Get new added Object
+    new_export = []
+    user_lvl_export = {}
     for obj_tuple in list(request.form):
+        # GET NEW LEVEL
+        if obj_tuple[0] == '{':
+            obj_j = obj_tuple.replace("'", "\"")
+            obj_dict = json.loads(obj_j)  # TODO sanitize
+            lvl = int(request.form.getlist(obj_tuple)[0])
+            user_lvl_export[f"{obj_dict['type']}:{obj_dict['subtype']}:{obj_dict['id']}"] = lvl
+
         l_input = request.form.getlist(obj_tuple)
         if len(l_input) == 3:
-            obj_type = l_input[0]
+            if l_input[0] != 'Object type...':
-            obj_id = l_input[1]
+                new_type, new_subtype = l_input[0].split(':', 1)
-            lvl = l_input[2]
+                if not new_subtype:
-            lvl = MispExport.sanitize_obj_export_lvl(lvl)
+                    new_subtype = ''
+                new_export.append({'type': new_type, 'subtype': new_subtype, 'id': l_input[1], 'lvl': l_input[2]})
 
-            obj_subtype = obj_type.split(';')
+    objects = []
-            if len(obj_subtype) == 2:
+    invalid_obj = []
-                obj_type = obj_subtype[0]
+    for obj in new_export:
-                obj_subtype = obj_subtype[1]
+        if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
+            invalid_obj.append(obj)
         else:
-                obj_subtype = None
+            objects.append(obj)
-
+    for obj in Export.get_user_misp_objects_to_export(user_id):
-            obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl}
+        if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
-            if obj_subtype:
+            invalid_obj.append(obj)
-                obj_dict['subtype'] = obj_subtype
-
-            if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
-                l_obj_to_export.append(obj_dict)
-                AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
         else:
-                if obj_id:
+            str_id = f"{obj['type']}:{obj['subtype']}:{obj['id']}"
-                    l_obj_invalid.append(obj_dict)
+            if str_id in user_lvl_export:
+                obj['lvl'] = user_lvl_export[str_id]
+            objects.append(obj)
+
+    if invalid_obj:
+        object_types = ail_objects.get_all_objects_with_subtypes_tuple()
+        return render_template("export_object.html", object_types=object_types,
+                               to_export=objects, l_obj_invalid=invalid_obj)
+
+    export = request.form.get('export_to_misp', False)
+    distribution = request.form.get('misp_event_distribution')
+    threat_level_id = request.form.get('threat_level_id')
+    analysis = request.form.get('misp_event_analysis')
+    info = request.form.get('misp_event_info')
+    publish = request.form.get('misp_event_info', False)
+
+    objs = ail_objects.get_objects(objects)
+    event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id,
+                                     analysis=analysis, info=info, export=export, publish=publish)
+
+    Export.delete_user_misp_objects_to_export(user_id)
+
+    if not export:
+        return send_file(io.BytesIO(event['event'].encode()), as_attachment=True,
+                         download_name=f'ail_export_{event["uuid"]}.json')
     else:
-            dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
+        object_types = ail_objects.get_all_objects_with_subtypes_tuple()
-
+        return render_template("export_object.html", object_types=object_types,
-    # print(dict_misp_event_export)
+                               misp_url=event['url'])
-
-    if dict_misp_event_export.get('export_to_misp', None):
-        export_to_misp = True
-    else:
-        dict_misp_event_export = None
-
-    if l_obj_invalid:
-        # get user saved obj to export # # TODO: # performance
-        l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
-
-        for obj_dict in l_obj_invalid: # set uuid input
-            obj_dict['uuid'] = str(uuid.uuid4())
-            subtype = obj_dict.get('subtype', None)
-            if subtype:
-                obj_dict['type'] = f'{obj_dict["type"]};{subtype}'
-
-        return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
-                               l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)
-    else:
-        if export_to_misp and MispExport.ping_misp():
-            event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event')
-
-            event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None),
-                                        threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None),
-                                        publish=dict_misp_event_export.get('misp_publish', None),
-                                        analysis=dict_misp_event_export.get('misp_event_analysis', None),
-                                        event_info=dict_misp_event_export.get('misp_event_info', None))
-
-            AILObjects.delete_all_user_object_to_export(user_id)
-            return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
-                                    event_metadata=event_metadata,
-                                    l_obj_invalid=[], dict_misp_event_export=[])
-        else:
-            # get user saved obj to export # # TODO: # performance
-            json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
-            export_filename = MispExport.get_export_filename(json_export)
-            json_export = MispExport.create_in_memory_file(json_export.to_json())
-            AILObjects.delete_all_user_object_to_export(user_id)
-            return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
 
 
-@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
+@import_export.route("/objects/misp/export/add", methods=['GET'])
 @login_required
 @login_analyst
 def add_object_id_to_export():
     user_id = current_user.get_id()
-    user_id = current_user.get_id()
+    obj_type = request.args.get('type')
-    obj_type = request.args.get('obj_type')
+    obj_id = request.args.get('id')
-    obj_id = request.args.get('obj_id')
+    obj_subtype = request.args.get('subtype')
-    obj_subtype = request.args.get('obj_subtype')
+    obj_lvl = request.args.get('lvl')
-    obj_lvl = request.args.get('obj_lvl')
+
-    AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
+    try:
+        obj_lvl = int(obj_lvl)
+    except:
+        obj_lvl = 0
+
+    if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
+        abort(404)
+    Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
     # redirect
-    return redirect(url_for('import_export.export_object'))
+    return redirect(url_for('import_export.objects_misp_export'))
+
+
+@import_export.route("/objects/misp/export/delete", methods=['GET'])
+@login_required
+@login_analyst
+def delete_object_id_to_export():
+    user_id = current_user.get_id()
+    obj_type = request.args.get('type')
+    obj_id = request.args.get('id')
+    obj_subtype = request.args.get('subtype')
+
+    Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
+    return jsonify(success=True)
+
 
 @import_export.route("/import_export/investigation", methods=['GET'])
 @login_required
 @login_analyst
 def export_investigation():
     investigation_uuid = request.args.get("uuid")
-
+    if Export.ping_misp():
-    if MispExport.ping_misp():
+        event = Export.create_investigation_misp_event(investigation_uuid)
-        event_metadata = MispExport.create_investigation_event(investigation_uuid)
+        print(event)
     else:
-        return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400
+        return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
-
+                        mimetype='application/json'), 400
     return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
-
-
-# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
-# @login_required
-# @login_analyst
-# def delete_object_id_to_export():
-#     user_id = current_user.get_id()
-#     obj_type = request.args.get('obj_type')
-#     obj_id = request.args.get('obj_id')
-#     obj_subtype = request.args.get('obj_subtype')
-#     obj_lvl = request.args.get('obj_lvl')
-#     AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
-#     # redirect
-#     return 'ok'

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
+"""
-    Flask functions and routes for the trending modules page
+    Flask functions and routes for the Item Submit modules page
-'''
+"""
 ##################################
 # Import External packages
 ##################################
 import re
 import os
 import sys
-import json
 import string
 import datetime
 import unicodedata
@@ -20,7 +19,7 @@ from io import BytesIO
 from functools import wraps
 
 # Flask
-from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
+from flask import render_template, jsonify, request, Blueprint, url_for, redirect, abort
 from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
@@ -28,56 +27,39 @@ from flask_login import login_required
 ##################################
 # Import Project packages
 ##################################
+from export import Export
 from lib import Tag
 from lib.objects.Items import Item
 
 from packages import Import_helper
 
-from pytaxonomies import Taxonomies
+from pytaxonomies import Taxonomies # TODO REMOVE ME
 
-try:
-    from pymisp.mispevent import MISPObject
-    flag_misp = True
-except:
-    flag_misp = False
-try:
-    from thehive4py.models import Case, CaseTask, CustomFieldHelper, CaseObservable
-    flag_hive = True
-except:
-    flag_hive = False
 
 # ============ VARIABLES ============
 import Flask_config
 
 app = Flask_config.app
 baseUrl = Flask_config.baseUrl
-r_serv_metadata = Flask_config.r_serv_metadata
+
-r_serv_db = Flask_config.r_serv_db
+
-r_serv_log_submit = Flask_config.r_serv_log_submit
+
+r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
+r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
+r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
 
 logger = Flask_config.redis_logger
 
-pymisp = Flask_config.pymisp
-if pymisp is False:
-    flag_misp = False
-
-HiveApi = Flask_config.HiveApi
-if HiveApi is False:
-    flag_hive = False
-
-PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
 
 valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
 
 UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
 
-misp_event_url = Flask_config.misp_event_url
-hive_case_url = Flask_config.hive_case_url
-
 text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000)
 file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
 allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
 
+PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
 
 # ============ Validators ============
 def limit_content_length():
@@ -117,133 +99,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
     # keep only whitelisted chars
     return ''.join(c for c in cleaned_filename if c in whitelist)
 
-def date_to_str(date):
-    return "{0}-{1}-{2}".format(date.year, date.month, date.day)
-
-def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path):
-
-    item = Item(path)
-    source = item.get_source()
-    ail_uuid = r_serv_db.get('ail:uuid')
-    pseudofile = BytesIO(item.get_content(binary=True))
-
-    temp = item.get_duplicates()
-
-    # beautifier
-    if not temp:
-        temp = ''
-
-    p_duplicate_number = len(temp) if len(temp) >= 0 else 0
-
-    to_ret = ""
-    for dup in temp[:10]:
-        dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
-        dup = json.loads(dup)
-        algo = dup[0]
-        path = dup[1].split('/')[-6:]
-        path = '/'.join(path)[:-3] # -3 removes .gz
-        if algo == 'tlsh':
-            perc = 100 - int(dup[2])
-        else:
-            perc = dup[2]
-        to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
-    p_duplicate = to_ret
-
-    today = datetime.date.today()
-    # [0-3]
-    if publish == 'True':
-        published = True
-    else:
-        published = False
-    org_id = None
-    orgc_id = None
-    sharing_group_id = None
-    date = today
-    event = pymisp.new_event(distribution, threat_level_id,
-            analysis, info, date,
-            published, orgc_id, org_id, sharing_group_id)
-    eventUuid = event['Event']['uuid']
-    eventid = event['Event']['id']
-
-    r_serv_metadata.set('misp_events:'+path, eventid)
-
-    # add tags
-    for tag in l_tags:
-        pymisp.tag(eventUuid, tag)
-
-    # create attributes
-    obj_name = 'ail-leak'
-    leak_obj = MISPObject(obj_name)
-    leak_obj.add_attribute('sensor', value=ail_uuid, type="text")
-    leak_obj.add_attribute('origin', value=source, type='text')
-    leak_obj.add_attribute('last-seen', value=date_to_str(item.get_date()), type='datetime')
-    leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment")
-
-    if p_duplicate_number > 0:
-        leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
-        leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
-
-    try:
-        templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name][0]
-    except IndexError:
-        valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
-        print (f"Template for type {obj_name} not found! Valid types are: {valid_types}")
-        return False
-    r = pymisp.add_object(eventid, templateID, leak_obj)
-    if 'errors' in r:
-        print(r)
-        return False
-    else:
-        event_url = misp_event_url + eventid
-        return event_url
-
-def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
-
-    ail_uuid = r_serv_db.get('ail:uuid')
-    source = path.split('/')[-6:]
-    source = '/'.join(source)[:-3]
-    # get item date
-    var = path.split('/')
-    last_seen = "{0}-{1}-{2}".format(var[-4], var[-3], var[-2])
-
-    case = Case(title=hive_case_title,
-                tlp=hive_tlp,
-                severity=threat_level,
-                flag=False,
-                tags=l_tags,
-                description='hive_description')
-
-    # Create the case
-    id = None
-    response = HiveApi.create_case(case)
-    if response.status_code == 201:
-        id = response.json()['id']
-
-        observ_sensor = CaseObservable(dataType="other", data=[ail_uuid], message="sensor")
-        observ_file = CaseObservable(dataType="file", data=[path], tags=l_tags)
-        observ_source = CaseObservable(dataType="other", data=[source], message="source")
-        observ_last_seen = CaseObservable(dataType="other", data=[last_seen], message="last-seen")
-
-        res = HiveApi.create_case_observable(id,observ_sensor)
-        if res.status_code != 201:
-            logger.info(f'ko sensor: {res.status_code}/{res.text}')
-        res = HiveApi.create_case_observable(id, observ_source)
-        if res.status_code != 201:
-            logger.info(f'ko source: {res.status_code}/{res.text}')
-        res = HiveApi.create_case_observable(id, observ_file)
-        if res.status_code != 201:
-            logger.info(f'ko file: {res.status_code}/{res.text}')
-        res = HiveApi.create_case_observable(id, observ_last_seen)
-        if res.status_code != 201:
-            logger.info(f'ko last_seen: {res.status_code}/{res.text}')
-
-        r_serv_metadata.set('hive_cases:'+path, id)
-
-        return hive_case_url.replace('id_here', id)
-    else:
-        logger.info(f'ko: {response.status_code}/{response.text}')
-        return False
-
 # ============= ROUTES ==============
 
 @PasteSubmit.route("/PasteSubmit/", methods=['GET'])
@@ -462,50 +317,31 @@ def submit_status():
     else:
         return 'INVALID UUID'
 
-
+# TODO MOVE ME IN import_export blueprint
-@PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
-@login_required
-@login_analyst
-def create_misp_event():
-
-    distribution = int(request.form['misp_data[Event][distribution]'])
-    threat_level_id = int(request.form['misp_data[Event][threat_level_id]'])
-    analysis = int(request.form['misp_data[Event][analysis]'])
-    info = request.form['misp_data[Event][info]']
-    path = request.form['paste']
-    publish = request.form.get('misp_publish')
-
-    #verify input
-    if (0 <= distribution <= 3) and (1 <= threat_level_id <= 4) and (0 <= analysis <= 2):
-
-        l_tags = list(r_serv_metadata.smembers('tag:'+path))
-        event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path)
-
-        if event != False:
-            return redirect(event)
-        else:
-            return 'error, event creation'
-    return 'error0'
-
 @PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
 @login_required
 @login_analyst
 def create_hive_case():
 
-    hive_tlp = int(request.form['hive_tlp'])
+    tlp = request.form['hive_tlp']
-    threat_level = int(request.form['threat_level_hive'])
+    if tlp:
-    hive_description = request.form['hive_description']
+        tlp = int(tlp)
-    hive_case_title = request.form['hive_case_title']
+    else:
-    path = os.environ['AIL_HOME'] + "/PASTES/"+ request.form['paste']
+        tlp = 2
+    threat_level = request.form['threat_level_hive']
+    if threat_level:
+        threat_level = int(threat_level)
+    else:
+        threat_level = 2
+    description = request.form['hive_description']
+    title = request.form['hive_case_title']
+    item_id = request.form['obj_id']
 
-    #verify input
+    if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
-    if (0 <= hive_tlp <= 3) and (1 <= threat_level <= 4):
 
-        l_tags = list(r_serv_metadata.smembers('tag:'+path))
+        case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
-        case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
+        if case_id:
-
+            return redirect(get_case_url(case_id))
-        if case != False:
-            return redirect(case)
         else:
             return 'error'
 

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -131,7 +131,7 @@ def get_all_types_id(correlation_type):
 
 def get_key_id_metadata(obj_type, subtype, obj_id):
     obj = ail_objects.get_object_meta(obj_type, subtype, obj_id)
-    return obj._get_meta()
+    return obj
 
 def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
     sparklines_value = []

var/www/modules/hashDecoded/templates/DaysCorrelation.html

@@ -171,7 +171,7 @@
 							{% for key_id in all_metadata %}
 									<tr>
 										<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td>
-										<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?object_type={{correlation_type_n}}&type_id={{ all_metadata[key_id]['type_id'] }}&correlation_id={{ key_id }}&correlation_objects=paste">{{ key_id }}</a></td>
+										<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type={{correlation_type_n}}&subtype={{ all_metadata[key_id]['type_id'] }}&id={{ key_id }}">{{ key_id }}</a></td>
 										<td>{{ all_metadata[key_id]['first_seen'] }}</td>
 										<td>{{ all_metadata[key_id]['last_seen'] }}</td>
 										<td>{{ all_metadata[key_id]['nb_seen'] }}</td>

var/www/modules/hashDecoded/templates/hashDecoded.html

@@ -158,7 +158,7 @@
 							{% for b64 in l_64 %}
 									<tr>
 										<td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
-										<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a></td>
+										<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=decoded&id={{ b64[2] }}">{{ b64[2] }}</a></td>
 										<td>{{ b64[5] }}</td>
 										<td>{{ b64[6] }}</td>
 										<td>{{ b64[3] }}</td>

var/www/templates/import_export/block_add_user_object_to_export.html

@@ -1,4 +1,4 @@
-<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
+<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
   Add to
   <img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
   Export

var/www/templates/import_export/block_add_user_object_to_export_small.html

@@ -1,6 +1,6 @@
 <div class="d-flex flex-row-reverse bd-highlight">
   <div>
-    <a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
+    <a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
       <img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
     </a>
   </div>

var/www/templates/import_export/block_to_export_input.html

@@ -1,38 +1,21 @@
 <div class="input-group mb-1">
-	<select class="custom-select col-2" name="{{input_uuid}}" value="{{obj_type}}">
+	<select class="custom-select col-2" name="{{input_uuid}}" disabled>
-		{% if not obj_type %}
+        <option selected="" value="{{ obj_type }}:{{ obj_subtype }}">
-			<option selected="">Object type...</option>
+            {% if obj_subtype %}
+                {{ obj_type }} - {{ obj_subtype }}
             {%  else %}
-			<option>Object type...</option>
+                {{ obj_type }}
             {% endif %}
-		<option value="item" {%if obj_type=="item"%}selected{%endif%}>Item</option>
+        </option>
-		<option value="domain" {%if obj_type=="domain"%}selected{%endif%}>Domain</option>
-		<option value="image" {%if obj_type=="image"%}selected{%endif%}>Image</option>
-		<option value="decoded" {%if obj_type=="decoded"%}selected{%endif%}>Decoded</option>
-		<option value="pgp;key" {%if obj_type=="pgp" and obj_subtype=="key"%}selected{%endif%}>PGP - Key</option>
-		<option value="pgp;name" {%if obj_type=="pgp" and obj_subtype=="name"%}selected{%endif%}>PGP - Name</option>
-		<option value="pgp;mail" {%if obj_type=="pgp" and obj_subtype=="mail"%}selected{%endif%}>PGP - Mail</option>
-		<option value="cryptocurrency;bitcoin" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin"%}selected{%endif%}>Cryptocurrency - Bitcoin</option>
-		<option value="cryptocurrency;bitcoin-cash" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin-cash"%}selected{%endif%}>Cryptocurrency - Bitcoin Cash</option>
-		<option value="cryptocurrency;dash" {%if obj_type=="cryptocurrency" and obj_subtype=="dash"%}selected{%endif%}>Cryptocurrency - Dash</option>
-		<option value="cryptocurrency;etherum" {%if obj_type=="cryptocurrency" and obj_subtype=="etherum"%}selected{%endif%}>Cryptocurrency - Etherum</option>
-		<option value="cryptocurrency;litecoin" {%if obj_type=="cryptocurrency" and obj_subtype=="litecoin"%}selected{%endif%}>Cryptocurrency - Litecoin</option>
-		<option value="cryptocurrency;monero" {%if obj_type=="cryptocurrency" and obj_subtype=="monero"%}selected{%endif%}>Cryptocurrency - Monero</option>
-		<option value="cryptocurrency;zcash" {%if obj_type=="cryptocurrency" and obj_subtype=="zcash"%}selected{%endif%}>Cryptocurrency - Zcash</option>
-
-		<option value="username;telegram" {%if obj_type=="username" and obj_subtype=="telegram"%}selected{%endif%}>Username - telegram</option>
-		<option value="username;twitter" {%if obj_type=="username" and obj_subtype=="twitter"%}selected{%endif%}>Username - twitter</option>
-		<option value="username;jabber" {%if obj_type=="username" and obj_subtype=="jabber"%}selected{%endif%}>Username - jabber</option>
-
 	</select>
-	<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}">
+	<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}" disabled>
 	<input class="form-control col-1" type="number" min="0" name="{{input_uuid}}" {%if obj_lvl%}value="{{obj_lvl}}"{%else%}value="0"{%endif%}>
-	<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>
+	<span class="btn btn-danger input-group-addon delete-field col-1" id="{{ obj_type }}:{{ obj_subtype }}:{{ obj_id }}"><i class="fas fa-trash-alt"></i></span>
 	<div class="invalid-feedback">
 		{%if obj_type=="Object type..."%}
 			&nbsp;&nbsp;&nbsp; Please select an object type.
 		{%else%}
-			&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Unknow or Invalid ID.
+			&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Unknown or Invalid ID.
 		{%endif%}
   </div>
 </div>

var/www/templates/import_export/export_object.html

@@ -34,7 +34,7 @@
 						</div>
 						<div class="card-body">
 
-							<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
+							<form action="{{ url_for('import_export.objects_misp_export_post') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
 
 								<div id="container-id-to-import">
 
@@ -53,30 +53,24 @@
                                                     <div class="input-group mb-1">
                                                         <select class="custom-select col-2" name="first_obj_to_export" id="obj_input_type">
                                                             <option selected>Object type...</option>
-													    <option value="item">Item</option>
+                                                            {% for object_type in object_types %}
-													    <option value="domain">Domain</option>
+                                                                <option value="{{ object_type[0] }}:{{ object_type[1] }}">
-													    <option value="image">Image</option>
+                                                                    {% if object_type[1] %}
-															<option value="decoded">Decoded</option>
+                                                                        {{ object_type[0] }} - {{ object_type[1] }}
-															<option value="pgp;key">PGP - Key</option>
+                                                                    {%  else %}
-															<option value="pgp;name">PGP - Name</option>
+                                                                        {{ object_type[0] }}
-															<option value="pgp;mail">PGP - Mail</option>
+                                                                    {% endif %}
-															<option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option>
+                                                                </option>
-															<option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option>
+                                                            {% endfor %}
-															<option value="cryptocurrency;dash">Cryptocurrency - Dash</option>
-															<option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option>
-															<option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option>
-															<option value="cryptocurrency;monero">Cryptocurrency - Monero</option>
-															<option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option>
-															<option value="username;telegram">Username - telegram</option>
-															<option value="username;twitter">Username - twitter</option>
-															<option value="username;jabber">Username - jabber</option>
                                                         </select>
+
                                                         <input type="text" class="form-control col-8" name="first_obj_to_export" id="obj_input_id">
 														<input class="form-control col-1" type="number" min="0" value="0" name="first_obj_to_export" id="obj_input_lvl">
 									                    <span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
 									                </div>
-													{% for obj_dict in l_obj_to_export %}
+
-														{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
+													{% for obj_dict in to_export %}
+														{% with obj_type=obj_dict['type'], obj_subtype=obj_dict['subtype'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
 															{% include 'import_export/block_to_export_input.html' %}
 														{% endwith %}
 													{% endfor %}
@@ -84,7 +78,7 @@
 													<br>
 
 													{% for obj_dict in l_obj_invalid %}
-														{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=True%}
+														{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=True%}
 															{% include 'import_export/block_to_export_input.html' %}
 														{% endwith %}
 													{% endfor %}
@@ -119,10 +113,10 @@
 						</div>
 					</div>
 
-					{% if event_metadata %}
+					{% if misp_url %}
 						MISP Event Created:
-						<a target="_blank" href="{{  event_metadata['url'] }}">
+						<a target="_blank" href="{{  misp_url }}">
-							{{  event_metadata['url'] }}
+							{{  misp_url }}
 						</a>
 					{% endif %}
 
@@ -155,7 +149,17 @@ function toggle_sidebar(){
 }
 
 var input_1 = '<div class="input-group mb-1"><select class="custom-select col-2" name="'
-var input_2 = '"><option selected>Object type...</option><option value="item">Item</option><option value="domain">Domain</option><option value="image">Image</option><option value="decoded">Decoded</option><option value="pgp;key">PGP - Key</option><option value="pgp;name">PGP - Name</option><option value="pgp;mail">PGP - Mail</option><option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option><option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option><option value="cryptocurrency;dash">Cryptocurrency - Dash</option><option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option><option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option><option value="cryptocurrency;monero">Cryptocurrency - Monero</option><option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option></select><input type="text" class="form-control col-8" name="'
+var input_2 = '"><option selected>Object type...</option>' +
+    {% for object_type in object_types %}
+        '<option value="{{ object_type[0] }}:{{ object_type[1] }}">' +
+        {% if object_type[1] %}
+            '{{ object_type[0] }} - {{ object_type[1] }}' +
+        {%  else %}
+            '{{ object_type[0] }}' +
+        {% endif %}
+        '</option>' +
+    {% endfor %}
+    '</select><input type="text" class="form-control col-8" name="'
 var input_3 = '"><input class="form-control col-1" type="number" min="0" value="0" name="'
 var input_4 = '"></div>';
 var minusButton = '<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>';
@@ -168,8 +172,10 @@ $('.add-field').click(function() {
 });
 
 $('.fields').on('click', '.delete-field', function(){
+    var obj = $(this).attr('id').split(":", 3);
+    var params = "?type=" + obj[0] + "&subtype=" + obj[1] +"&id=" + obj[2];
+    $.ajax({url: "{{ url_for('import_export.delete_object_id_to_export') }}" + params })
     $(this).parent().remove();
-	//$.get( "#")
 });
 
 

var/www/templates/investigations/view_investigation.html

@@ -107,8 +107,8 @@
 
 									{% if metadata['misp_events'] %}
 										{% for misp_url in metadata['misp_events'] %}
-											<a target="_blank" href="{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}">
+											<a target="_blank" href="{{ misp_url }}">
-												{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}</br>
+												{{ misp_url }}</br>
 											</a>
 										{% endfor %}
 									{% endif %}

var/www/templates/modals/create_hive_case.html

@@ -6,10 +6,10 @@
                 <img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
             </div>
 
-      <div class="modal-body">
-
             <form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
 
+            <div class="modal-body">
+
                     <div class="input clear required">
                         <label for="EventThreatLevelId">Threat Level</label>
                         <select name="threat_level_hive" id="EventThreatLevelId" required="required">
@@ -19,7 +19,7 @@
                         </select>
                     </div>
                     <div class="input select required">
-          <label for="TLP">TLP</label>
+                        <label for="hive_tlp">TLP</label>
                         <select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
                             <option value="0">White</option>
                             <option value="1">Green</option>
@@ -27,18 +27,15 @@
                             <option value="3">Red</option>
                         </select>
                     </div>
-        <div class="clear required">
+                    <div class="clear">
                         <label for="hive_case_title">Title</label>
-          <input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
+                        <input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title"/>
                     </div>
-        <div class="clear required">
+                    <div class="clear">
                         <label for="hive_description">Description</label>
-          <input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
+                        <input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
                     </div>
-        <input type="hidden" id="paste" name="paste" value="{{ dict_item['id'] }}">
+                    <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
-
-
-
             </div>
 
             <div class="modal-footer">
@@ -47,74 +44,11 @@
                     Create Case
                 </button>
 
-        </form>
                 <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
             </div>
 
+            </form>
+
         </div>
     </div>
 </div>
-
-<script> // TODO: add tags to PASTE or DOMAIN
-var ltags;
-var ltagsgalaxies;
-
-$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
-    function(data) {
-
-      ltags = $('#ltags').tagSuggest({
-        data: data,
-        maxDropHeight: 200,
-        name: 'ltags'
-      });
-    });
-
-$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
-  function(data) {
-
-    ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
-      data: data,
-      maxDropHeight: 200,
-      name: 'ltagsgalaxies'
-    });
-  });
-
-jQuery("#all-tags-taxonomies").click(function(e){
-        //change input tags list
-      $.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
-        function(data) {
-          ltags.setData(data)
-      });
-    });
-
-jQuery("#all-tags-galaxies").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
-        function(data) {
-          ltagsgalaxies.setData(data)
-      });
-    });
-
-{% for taxo in modal_add_tags['active_taxonomies'] %}
-    jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
-        function(data) {
-          ltags.setData(data)
-  });
-});
-{% endfor %}
-
-{% for galaxy in modal_add_tags['active_galaxies'] %}
-    jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
-      $.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
-        function(data) {
-          ltagsgalaxies.setData(data)
-  });
-});
-{% endfor %}
-
-function addTags() {
-  var tags = ltags.getValue()
-  var tagsgalaxy = ltagsgalaxies.getValue()
-  window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}");
-}
-</script>

var/www/templates/sidebars/sidebar_objects.html

@@ -75,7 +75,7 @@
       	      </a>
       	    </li>
             <li class="nav-item">
-              <a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
+              <a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export">
                 <b>Export</b>
               </a>
             </li>