chg: [exporter] refactor + add a new way to create and use exporters

bin/export/Export.py

@@ -199,145 +199,7 @@ def get_item_hive_cases(item_id):
 #           MISP
 ##################################
 
-try:
+#####################################################################3
-    from mispKEYS import misp_url, misp_key, misp_verifycert
-
-    MISP_URL = misp_url
-    MISP_KEY = misp_key
-    MISP_VERIFYCERT = misp_verifycert
-    if MISP_URL.endswith('/'):
-        MISP_URL = MISP_URL[:-1]
-except:
-    MISP_URL = None
-    MISP_KEY = None
-    MISP_VERIFYCERT = None
-
-
-def get_misp_client():
-    return PyMISP(misp_url, misp_key, misp_verifycert)
-
-
-# # TODO: return error
-def ping_misp():
-    try:
-        PyMISP(misp_url, misp_key, misp_verifycert)
-        return True
-    except Exception as e:
-        print(e)
-        return False
-
-
-def sanitize_misp_event_distribution(distribution):
-    try:
-        int(distribution)
-        if 0 <= distribution <= 3:
-            return distribution
-        else:
-            return 0
-    except:
-        return 0
-
-
-def sanitize_misp_event_threat_level(threat_level):
-    try:
-        int(threat_level)
-        if 1 <= threat_level <= 4:
-            return threat_level
-        else:
-            return 4
-    except:
-        return 4
-
-
-def sanitize_misp_event_analysis(analysis):
-    try:
-        int(analysis)
-        if 0 <= analysis <= 2:
-            return analysis
-        else:
-            return 0
-    except:
-        return 0
-
-def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0,
-                      info=None, tags=None, export=False):
-    if tags is None:
-        tags = []
-    event = MISPEvent()
-    if not event_uuid:
-        event_uuid = str(uuid.uuid4())
-    event.uuid = event_uuid
-    if date:
-        event.date = date
-    if not info:
-        info = 'AIL framework export'
-    event.info = info
-    if publish:
-        event.publish()
-    for tag in tags:
-        event.add_tag(tag)
-    event.distribution = sanitize_misp_event_distribution(distribution)
-    event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id)
-    event.analysis = sanitize_misp_event_analysis(analysis)
-
-    misp_objects = ail_objects.get_misp_objects(objs)
-    for obj in misp_objects:
-        event.add_object(obj)
-
-    # print(event.to_json())
-
-    if export:
-        misp = get_misp_client()
-        misp_event = misp.add_event(event)
-        # TODO: handle error
-
-        misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}'
-        return misp_event
-    else:
-        return {'uuid': event['uuid'], 'event': event.to_json()}
-
-def create_investigation_misp_event(investigation_uuid):
-    investigation = Investigation(investigation_uuid)
-    objs = ail_objects.get_objects(investigation.get_objects())
-
-    event = create_misp_event(objs,
-                              # event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ????
-                              date=investigation.get_date(),
-                              distribution=0,
-                              threat_level_id=investigation.get_threat_level(),
-                              analysis=investigation.get_analysis(),
-                              info=investigation.get_info(),
-                              tags=investigation.get_tags(),
-                              export=True)
-    url = event['url']
-    if url:
-        investigation.add_misp_events(url)
-    return url
-
-def get_user_misp_objects_to_export(user_id):
-    objs = []
-    objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
-    for obj in objects:
-        obj_type, obj_subtype, obj_id = obj.split(':', 2)
-        lvl = objects[obj]
-        try:
-            lvl = int(lvl)
-        except:
-            lvl = 0
-        objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
-    return objs
-
-def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
-    if not obj_subtype:
-        obj_subtype = ''
-    r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
-
-def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
-    r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
-
-def delete_user_misp_objects_to_export(user_id):
-    r_db.delete(f'user:obj:misp:export:{user_id}')
-
 
 ###########################################################
 # # set default

bin/exporter/MISPExporter.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Importer Class
+================
+
+Import Content
+
+"""
+import os
+import sys
+import uuid
+
+from abc import ABC
+
+from pymisp import MISPEvent, PyMISP
+from urllib3 import disable_warnings as urllib3_disable_warnings
+
+sys.path.append('../../configs/keys')
+sys.path.append(os.environ['AIL_BIN'])
+#################################
+# Import Project packages
+#################################
+from exporter.abstract_exporter import AbstractExporter
+from lib.ConfigLoader import ConfigLoader
+from lib.Investigations import Investigation
+from lib.objects.abstract_object import AbstractObject
+
+# from lib.Tracker import Tracker
+
+config_loader = ConfigLoader()
+r_db = config_loader.get_db_conn("Kvrocks_DB")
+config_loader = None
+
+#### FUNCTIONS ####
+
+def get_user_misp_objects_to_export(user_id):
+    objs = []
+    objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
+    for obj in objects:
+        obj_type, obj_subtype, obj_id = obj.split(':', 2)
+        lvl = objects[obj]
+        try:
+            lvl = int(lvl)
+        except(TypeError, ValueError):
+            lvl = 0
+        objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
+    return objs
+
+def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
+    if not obj_subtype:
+        obj_subtype = ''
+    r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
+
+def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
+    r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
+
+def delete_user_misp_objects_to_export(user_id):
+    r_db.delete(f'user:obj:misp:export:{user_id}')
+
+# --- FUNCTIONS --- #
+
+# MISPExporter -> return correct exporter by type ????
+class MISPExporter(AbstractExporter, ABC): # <- AbstractMISPExporter ???????
+    """MISP Exporter
+
+    :param url: URL of the MISP instance you want to connect to
+    :param key: API key of the user you want to use
+    :param ssl: can be True or False (to check or to not check the validity of the certificate.
+    Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
+    """
+
+    def __init__(self, url='', key='', ssl=False):
+        super().__init__()
+
+        if url and key:
+            self.url = url
+            self.key = key
+            self.ssl = ssl
+            if self.ssl is False:
+                urllib3_disable_warnings()
+        elif url or key:
+            raise Exception('Error: missing url or api key')
+        else:
+            try:
+                from mispKEYS import misp_url, misp_key, misp_verifycert
+                self.url = misp_url
+                self.key = misp_key
+                self.ssl = misp_verifycert
+                if self.ssl is False:
+                    urllib3_disable_warnings()
+                if self.url.endswith('/'):
+                    self.url = self.url[:-1]
+            except Exception:  # ModuleNotFoundError
+                self.url = None
+                self.key = None
+                self.ssl = None
+
+    def get_misp(self):
+        return PyMISP(self.url, self.key, self.ssl)
+
+    # TODO catch exception
+    def get_misp_uuid(self):
+        misp = self.get_misp()
+        misp_setting = misp.get_server_setting('MISP.uuid')
+        return misp_setting.get('value')
+
+    # TODO ADD TIMEOUT
+    # TODO return error
+    def ping_misp(self):
+        try:
+            self.get_misp()
+            return True
+        except Exception as e:
+            print(e)
+            return False
+
+    @staticmethod
+    def sanitize_distribution(distribution):
+        try:
+            int(distribution)
+            if 0 <= distribution <= 3:
+                return distribution
+            else:
+                return 0
+        except (TypeError, ValueError):
+            return 0
+
+    @staticmethod
+    def sanitize_threat_level(threat_level):
+        try:
+            int(threat_level)
+            if 1 <= threat_level <= 4:
+                return threat_level
+            else:
+                return 4
+        except (TypeError, ValueError):
+            return 4
+
+    @staticmethod
+    def sanitize_analysis(analysis):
+        try:
+            int(analysis)
+            if 0 <= analysis <= 2:
+                return analysis
+            else:
+                return 0
+        except (TypeError, ValueError):
+            return 0
+
+    # TODO EVENT REPORT ???????
+    def create_event(self, objs, export=False, event_uuid=None, date=None, publish=False, info=None, tags=None,
+                     analysis=0, distribution=0, threat_level=4):
+        if tags is None:
+            tags = []
+        event = MISPEvent()
+        if not event_uuid:
+            event_uuid = str(uuid.uuid4())
+        event.uuid = event_uuid
+        if date:
+            event.date = date
+        if not info:
+            info = 'AIL framework export'
+        event.info = info
+        if publish:
+            event.publish()
+        for tag in tags:
+            event.add_tag(tag)
+        event.distribution = self.sanitize_distribution(distribution)
+        event.threat_level_id = self.sanitize_threat_level(threat_level)
+        event.analysis = self.sanitize_analysis(analysis)
+
+        misp_objects = ail_objects.get_misp_objects(objs)
+        for obj in misp_objects:
+            event.add_object(obj)
+        # print(event.to_json())
+
+        if export:
+            misp = self.get_misp()
+            misp_event = misp.add_event(event)
+            # TODO: handle error
+
+            misp_event['url'] = f'{self.url}/events/view/{misp_event["Event"]["uuid"]}'
+            return misp_event
+        else:
+            return event.to_json()
+            # return {'uuid': event['uuid'], 'event': event.to_json()}
+
+        # EXPORTER CHAIN
+        # if self.chainable
+        # if self.next_exporter:
+        #     next_exporter.export({'type': 'misp_event', 'data': {'event': misp_event}})
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}(url={self.url})'
+
+class MISPExporterAILObjects(MISPExporter):
+    """MISPExporter AILObjects
+
+    :param url: URL of the MISP instance you want to connect to
+    :param key: API key of the user you want to use
+    :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
+    """
+
+    def __init__(self, url='', key='', ssl=False):
+        super().__init__(url=url, key=key, ssl=ssl)
+
+    def export(self, objects, export=False, event_uuid=None, date=None, publish=False, info=None, tags=[],
+               analysis=0, distribution=0, threat_level=4):
+        """Export a list of AILObjects as a MISP event
+
+        :param objects: Investigation object or investigation uuid string
+        :type objects: list[AbstractObject]
+        """
+        # objects ????
+        # TODO convert string tuple to object
+
+        return self.create_event(objects, event_uuid=event_uuid, date=date, publish=publish,
+                                 analysis=analysis, distribution=distribution, threat_level=threat_level,
+                                 info=info, tags=tags, export=export)
+
+class MISPExporterInvestigation(MISPExporter):
+    """MISPExporter Investigation
+
+    :param url: URL of the MISP instance you want to connect to
+    :param key: API key of the user you want to use
+    :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
+    """
+
+    def __init__(self, url='', key='', ssl=False):
+        super().__init__(url=url, key=key, ssl=ssl)
+
+    def export(self, investigation):
+        """Export an Investigation as a MISP event
+
+        :param investigation: Investigation object or investigation uuid string
+        :type investigation: Investigation | str
+        """
+        if not isinstance(investigation, Investigation):
+            investigation = Investigation(investigation)
+        objs = ail_objects.get_objects(investigation.get_objects())
+        event = self.create_event(objs,
+                                  date=investigation.get_date(),
+                                  distribution=0,
+                                  threat_level=investigation.get_threat_level(),
+                                  analysis=investigation.get_analysis(),
+                                  info=investigation.get_info(),
+                                  tags=investigation.get_tags(),
+                                  export=True)
+        url = event['url']
+        if url:
+            investigation.add_misp_events(url)
+        return url
+
+class MISPExporterTrackerMatch(MISPExporter):
+    """MISPExporter Tracker match
+
+    :param url: URL of the MISP instance you want to connect to
+    :param key: API key of the user you want to use
+    :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
+    """
+
+    def __init__(self, url='', key='', ssl=False):
+        super().__init__(url=url, key=key, ssl=ssl)
+
+    # TODO
+    def export(self, tracker, item):
+        pass
+
+
+if __name__ == '__main__':
+    exporter = MISPExporterAILObjects()
+
+    # from lib.objects.Cves import Cve
+    # from lib.objects.Items import Item
+    # objs_t = [Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
+    #           Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
+    #           Cve('CVE-2015-0410')]
+    # r = exporter.export(objs_t, export=False)
+    # print(r)
+
+    r = exporter.get_misp_uuid()
+    # r = misp.server_settings()
+    # for item in r['finalSettings']:
+    #     print()
+    #     print(item)
+    #     # print(r['finalSettings'][item])
+    #     # print()
+    #     print()
+
+    print(r)

bin/exporter/MailExporter.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Importer Class
+================
+
+Import Content
+
+"""
+import os
+import sys
+
+from abc import ABC
+
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+# from flask import escape
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from exporter.abstract_exporter import AbstractExporter
+from lib.ConfigLoader import ConfigLoader
+# from lib.objects.abstract_object import AbstractObject
+# from lib.Tracker import Tracker
+
+
+class MailExporter(AbstractExporter, ABC):
+    def __init__(self, host=None, port=None, password=None, user='', sender=''):
+        super().__init__()
+        config_loader = ConfigLoader()
+
+        if host:
+            self.host = host
+            self.port = port
+        else:
+            self.host = config_loader.get_config_str("Notifications", "sender_host")
+            self.port = config_loader.get_config_int("Notifications", "sender_port")
+        if password:
+            self.pw = password
+        else:
+            self.pw = config_loader.get_config_str("Notifications", "sender_pw")
+            if self.pw == 'None':
+                self.pw = None
+        if user:
+            self.user = user
+        else:
+            self.user = config_loader.get_config_str("Notifications", "sender_user")
+        if sender:
+            self.sender = sender
+        else:
+            self.sender = config_loader.get_config_str("Notifications", "sender")
+
+        # raise an exception if any of these is None
+        if (self.sender is None or
+                self.host is None or
+                self.port is None):
+            raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
+
+    def import(self):
+        pass
+
+    def get_smtp_client(self):
+        # try:
+        if self.pw is not None:
+            try:
+                smtp_server = smtplib.SMTP(self.host, self.port)
+                smtp_server.starttls()
+            except smtplib.SMTPNotSupportedError:
+                print("The server does not support the STARTTLS extension.")
+                smtp_server = smtplib.SMTP_SSL(self.host, self.port)
+
+            smtp_server.ehlo()
+            if self.user is not None:
+                smtp_server.login(self.user, self.pw)
+            else:
+                smtp_server.login(self.sender, self.pw)
+        else:
+            smtp_server = smtplib.SMTP(self.host, self.port)
+        return smtp_server
+        # except Exception as err:
+        # traceback.print_tb(err.__traceback__)
+        # publisher.warning(err)
+
+    def _export(self, recipient, subject, body):
+        mime_msg = MIMEMultipart()
+        mime_msg['From'] = self.sender
+        mime_msg['To'] = recipient
+        mime_msg['Subject'] = subject
+        mime_msg.attach(MIMEText(body, 'plain'))
+
+        # try:
+        smtp_client = self.get_smtp_client()
+        smtp_client.sendmail(self.sender, recipient, mime_msg.as_string())
+        smtp_client.quit()
+        # except Exception as err:
+        # traceback.print_tb(err.__traceback__)
+        # publisher.warning(err)
+        print(f'Send notification: {subject} to {recipient}')
+
+class MailExporterTracker(MailExporter):
+
+    def __init__(self, host=None, port=None, password=None, user='', sender=''):
+        super().__init__(host=host, port=port, password=password, user=user, sender=sender)
+
+    def export(self, tracker, obj):  # TODO match
+        tracker_type = tracker.get_type()
+        tracker_name = tracker.get_tracker()
+        subject = f'AIL Framework Tracker: {tracker_name}'  # TODO custom subject
+        body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
+        body += f'Item: {obj.id}\nurl:{obj.get_link()}'
+
+        # TODO match option
+        # if match:
+        #     body += f'Tracker Match:\n\n{escape(match)}'
+
+        for mail in tracker.get_mails():
+            self._export(mail, subject, body)

bin/exporter/WebHookExporter.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Importer Class
+================
+
+Import Content
+
+"""
+import os
+import requests
+import sys
+
+from abc import ABC
+
+sys.path.append(os.environ['AIL_BIN'])
+#################################
+# Import Project packages
+#################################
+from exporter.abstract_exporter import AbstractExporter
+
+# from ConfigLoader import ConfigLoader
+# from lib.objects.abstract_object import AbstractObject
+# from lib.Tracker import Tracker
+
+class WebHookExporter(AbstractExporter, ABC):
+    def __init__(self, url=''):
+        super().__init__()
+        self.url = url
+
+    def set_url(self, url):
+        self.url = url
+
+    def _export(self, data):
+        try:
+            response = requests.post(self.url, json=data)
+            if response.status_code >= 400:
+                print(f"Webhook request failed for {self.url}\nReason: {response.reason}")
+                # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
+        except Exception as e:
+            print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
+            # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
+
+
+class WebHookExporterTracker(WebHookExporter):
+
+    def __init__(self, url=''):
+        super().__init__(url=url)
+
+    # TODO Change exported keys
+    def export(self, tracker, obj):
+        self.set_url(tracker.get_webhook())
+        data = {'trackerId': tracker.get_uuid(),
+                'trackerType': tracker.get_type(),
+                'tags': tracker.get_tags(),
+                'tracker': tracker.get_tracker(),
+                # object
+                'itemId': obj.get_id(),
+                'itemURL': obj.get_link()}
+        # Item
+        # data['itemDate'] = obj.get_date()
+        # data["itemSource"] = obj.get_source()
+
+        self._export(data)

bin/exporter/abstract_exporter.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Importer Class
+================
+
+Import Content
+
+"""
+import os
+import sys
+
+from abc import ABC, abstractmethod
+
+
+# sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+# from ConfigLoader import ConfigLoader
+
+class AbstractExporter(ABC):
+    def __init__(self):
+        """
+        Init Module
+        """
+        # Module name if provided else instance className
+        self.name = self._name()
+
+    def _name(self):
+        """
+        Returns the instance class name (ie. the Exporter Name)
+        """
+        return self.__class__.__name__
+
+    @abstractmethod
+    def export(self, *args, **kwargs):
+        """Importer function"""
+        pass
+    #     res = self.export(*args, **kwargs)
+    #     if self.next_exporter:
+    #         self.next_exporter.exporter(res)
+
+

bin/importer/FeederImporter.py

@@ -27,6 +27,23 @@ r_db = config_loader.get_db_conn('Kvrocks_DB')
 config_loader = None
 # --- CONFIG --- #
 
+#### FUNCTIONS ####
+
+def add_json_feeder_to_queue(json_data):
+    json_data = json.dumps(json_data)
+    return r_db.rpush('importer:feeder', json_data)
+
+def api_add_json_feeder_to_queue(json_data):
+    if not json_data:
+        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
+    # # TODO: add JSON verification
+    res = add_json_feeder_to_queue(json_data)
+    if not res:
+        return {'status': 'error'}, 400
+    return {'status': 'success'}, 200
+
+# --- FUNCTIONS --- #
+
 class FeederImporter(AbstractImporter):
     def __init__(self):
         super().__init__()
@@ -89,9 +106,8 @@ class FeederModuleImporter(AbstractModule):
         self.importer = FeederImporter()
 
     def get_message(self):
-        return self.r_db.lpop('importer:feeder')  # TODO CHOOSE DB
+        return self.r_db.lpop('importer:feeder')
-        # TODO RELOAD LIST
+        # TODO RELOAD LIST after delta
-        # after delta
 
     def compute(self, message):
         # TODO HANDLE Invalid JSON
@@ -104,20 +120,6 @@ class FeederModuleImporter(AbstractModule):
     # server_cache.hincrby("mixer_cache:list_feeder", feeder_name, 1)
 
 
-def add_json_feeder_to_queue(json_data):
-    json_data = json.dumps(json_data)
-    return r_db.rpush('importer:feeder', json_data)
-
-def api_add_json_feeder_to_queue(json_data):
-    if not json_data:
-        return {'status': 'error', 'reason': 'Malformed JSON'}, 400
-    # # TODO: add JSON verification
-    res = add_json_feeder_to_queue(json_data)
-    if not res:
-        return {'status': 'error'}, 400
-    return {'status': 'success'}, 200
-
-
 # Launch Importer
 if __name__ == '__main__':
     module = FeederModuleImporter()

bin/importer/ZMQImporter.py

@@ -9,7 +9,6 @@ Import Content
 """
 import os
 import sys
-import time
 
 import zmq
 
@@ -63,6 +62,7 @@ class ZMQModuleImporter(AbstractModule):
         # TODO register all Importers
         self.zmq_importer.add(address, channel)
 
+    # TODO MESSAGE SOURCE - UI
     def get_message(self):
         for message in self.zmq_importer.importer():
             # remove channel from message

bin/lib/Investigations.py

@@ -91,6 +91,9 @@ class Investigation(object):
     def __init__(self, investigation_uuid):
         self.uuid = investigation_uuid
 
+    def exists(self):
+        return r_tracking.exists(f'investigations:data:{self.uuid}')
+
     def get_uuid(self, separator=False):
         if separator:
             return uuid.UUID(hex=self.uuid, version=4)

bin/trackers/Tracker_Regex.py

@@ -11,7 +11,6 @@ It processes every item coming from the global module and test the regex
 import os
 import sys
 import time
-import requests
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
@@ -22,7 +21,8 @@ from lib.objects.Items import Item
 from packages import Term
 from lib import Tracker
 
-import NotificationHelper
+from exporter.MailExporter import MailExporterTracker
+from exporter.WebHookExporter import WebHookExporterTracker
 
 class Tracker_Regex(AbstractModule):
 
@@ -38,12 +38,14 @@ class Tracker_Regex(AbstractModule):
 
         self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
 
-        self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
-
         # refresh Tracked Regex
         self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
         self.last_refresh = time.time()
 
+        # Exporter
+        self.exporters = {'mail': MailExporterTracker(),
+                          'webhook': WebHookExporterTracker()}
+
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
     def compute(self, item_id):
@@ -56,60 +58,45 @@ class Tracker_Regex(AbstractModule):
 
         item = Item(item_id)
         item_id = item.get_id()
-        item_content = item.get_content()
+        content = item.get_content()
 
         for regex in self.dict_regex_tracked:
-            matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, item_content)
+            matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content)
             if matched:
                 self.new_tracker_found(regex, 'regex', item)
 
-    def new_tracker_found(self, tracker, tracker_type, item):
+            # match = self.regex_finditer(self.dict_regex_tracked[regex], item_id, content)
-        uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type)
+            # if match:
+            #     self.new_tracker_found(regex, 'regex', item)
+
+    def new_tracker_found(self, tracker_name, tracker_type, item):
+        uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
 
         item_id = item.get_id()
-        item_date = item.get_date()
+        # date = item.get_date()
         item_source = item.get_source()
-        print(f'new tracked regex found: {tracker} in {item_id}')
+        print(f'new tracked regex found: {tracker_name} in {item_id}')
 
         for tracker_uuid in uuid_list:
+            tracker = Tracker.Tracker(tracker_uuid)
+
             # Source Filtering
-            tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
+            tracker_sources = tracker.get_sources()
             if tracker_sources and item_source not in tracker_sources:
                 continue
 
-            Tracker.add_tracked_item(tracker_uuid, item_id)
+            Tracker.add_tracked_item(tracker_uuid, item_id)  # TODO
 
-            tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
+            for tag in tracker.get_tags():
-            for tag in tags_to_add:
                 msg = f'{tag};{item_id}'
                 self.send_message_to_queue(msg, 'Tags')
 
-            mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
+            if tracker.mail_export():
-            if mail_to_notify:
+                # TODO add matches + custom subjects
-                mail_subject = Tracker.get_email_subject(tracker_uuid)
+                self.exporters['mail'].export(tracker, item)
-                mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
-                for mail in mail_to_notify:
-                    NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
 
-            # Webhook
+            if tracker.webhook_export():
-            webhook_to_post = Term.get_term_webhook(tracker_uuid)
+                self.exporters['webhook'].export(tracker, item)
-            if webhook_to_post:
-                json_request = {"trackerId": tracker_uuid,
-                                "itemId": item_id,
-                                "itemURL": self.full_item_url + item_id,
-                                "tracker": tracker,
-                                "itemSource": item_source,
-                                "itemDate": item_date,
-                                "tags": tags_to_add,
-                                "emailNotification": f'{mail_to_notify}',
-                                "trackerType": tracker_type
-                                }
-                try:
-                    response = requests.post(webhook_to_post, json=json_request)
-                    if response.status_code >= 400:
-                        self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
-                except:
-                    self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
 
 
 if __name__ == "__main__":

var/www/blueprints/import_export.py

@@ -1,17 +1,16 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
+"""
     Blueprint Flask: MISP format import export
-'''
+"""
 import io
 import os
 import sys
-import uuid
 import json
 
 from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort
-from flask_login import login_required, current_user, login_user, logout_user
+from flask_login import login_required, current_user
 
 sys.path.append('modules')
 
@@ -22,20 +21,22 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from export import Export
+from exporter import MISPExporter
 from lib.objects import ail_objects
+from lib.Investigations import Investigation
 
+# TODO REMOVE ME
+from export import Export  # TODO REMOVE ME
 from export import MispImport  # TODO REMOVE ME
-
 # TODO REMOVE ME
 
 # ============ BLUEPRINT ============
 import_export = Blueprint('import_export', __name__,
                           template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
 
-
 # ============ VARIABLES ============
-
+misp_exporter_objects = MISPExporter.MISPExporterAILObjects()
+misp_exporter_investigation = MISPExporter.MISPExporterInvestigation()
 
 # ============ FUNCTIONS ============
 
@@ -47,6 +48,7 @@ import_export = Blueprint('import_export', __name__,
 def import_object():
     return render_template("import_object.html")
 
+
 # TODO
 @import_export.route("/import_export/import_file", methods=['POST'])
 @login_required
@@ -87,7 +89,7 @@ def import_object_file():
 def objects_misp_export():
     user_id = current_user.get_id()
     object_types = ail_objects.get_all_objects_with_subtypes_tuple()
-    to_export = Export.get_user_misp_objects_to_export(user_id)
+    to_export = MISPExporter.get_user_misp_objects_to_export(user_id)
     return render_template("export_object.html", object_types=object_types, to_export=to_export)
 
 
@@ -123,7 +125,7 @@ def objects_misp_export_post():
             invalid_obj.append(obj)
         else:
             objects.append(obj)
-    for obj in Export.get_user_misp_objects_to_export(user_id):
+    for obj in MISPExporter.get_user_misp_objects_to_export(user_id):
         if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
             invalid_obj.append(obj)
         else:
@@ -139,16 +141,16 @@ def objects_misp_export_post():
 
     export = request.form.get('export_to_misp', False)
     distribution = request.form.get('misp_event_distribution')
-    threat_level_id = request.form.get('threat_level_id')
+    threat_level = request.form.get('threat_level_id')
     analysis = request.form.get('misp_event_analysis')
     info = request.form.get('misp_event_info')
     publish = request.form.get('misp_event_info', False)
 
     objs = ail_objects.get_objects(objects)
-    event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id,
+    event = misp_exporter_objects.create_event(objs, distribution=distribution, threat_level=threat_level,
                                                analysis=analysis, info=info, export=export, publish=publish)
 
-    Export.delete_user_misp_objects_to_export(user_id)
+    MISPExporter.delete_user_misp_objects_to_export(user_id)
 
     if not export:
         return send_file(io.BytesIO(event['event'].encode()), as_attachment=True,
@@ -176,7 +178,7 @@ def add_object_id_to_export():
 
     if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
         abort(404)
-    Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
+    MISPExporter.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
     # redirect
     return redirect(url_for('import_export.objects_misp_export'))
 
@@ -190,7 +192,7 @@ def delete_object_id_to_export():
     obj_id = request.args.get('id')
     obj_subtype = request.args.get('subtype')
 
-    Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
+    MISPExporter.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
     return jsonify(success=True)
 
 
@@ -199,8 +201,11 @@ def delete_object_id_to_export():
 @login_analyst
 def export_investigation():
     investigation_uuid = request.args.get("uuid")
-    if Export.ping_misp():
+    investigation = Investigation(investigation_uuid)
-        event = Export.create_investigation_misp_event(investigation_uuid)
+    if not investigation.exists():
+        abort(404)
+    if misp_exporter_objects.ping_misp():
+        event = misp_exporter_objects.export({'type': 'investigation', 'data': {'investigation': investigation}})
         print(event)
     else:
         return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
@@ -218,11 +223,12 @@ def create_thehive_case():
     tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
     item_id = request.form['obj_id']
 
-    item = Item(item_id)
+    item = ail_objects.get_object('item', '', item_id)
     if not item.exists():
         abort(404)
 
-    case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
+    case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level,
+                                         description=description)
     if case_id:
         return redirect(Export.get_case_url(case_id))
     else: