diff --git a/bin/export/Export.py b/bin/export/Export.py index 82c946e8..a3bd7305 100755 --- a/bin/export/Export.py +++ b/bin/export/Export.py @@ -199,145 +199,7 @@ def get_item_hive_cases(item_id): # MISP ################################## -try: - from mispKEYS import misp_url, misp_key, misp_verifycert - - MISP_URL = misp_url - MISP_KEY = misp_key - MISP_VERIFYCERT = misp_verifycert - if MISP_URL.endswith('/'): - MISP_URL = MISP_URL[:-1] -except: - MISP_URL = None - MISP_KEY = None - MISP_VERIFYCERT = None - - -def get_misp_client(): - return PyMISP(misp_url, misp_key, misp_verifycert) - - -# # TODO: return error -def ping_misp(): - try: - PyMISP(misp_url, misp_key, misp_verifycert) - return True - except Exception as e: - print(e) - return False - - -def sanitize_misp_event_distribution(distribution): - try: - int(distribution) - if 0 <= distribution <= 3: - return distribution - else: - return 0 - except: - return 0 - - -def sanitize_misp_event_threat_level(threat_level): - try: - int(threat_level) - if 1 <= threat_level <= 4: - return threat_level - else: - return 4 - except: - return 4 - - -def sanitize_misp_event_analysis(analysis): - try: - int(analysis) - if 0 <= analysis <= 2: - return analysis - else: - return 0 - except: - return 0 - -def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0, - info=None, tags=None, export=False): - if tags is None: - tags = [] - event = MISPEvent() - if not event_uuid: - event_uuid = str(uuid.uuid4()) - event.uuid = event_uuid - if date: - event.date = date - if not info: - info = 'AIL framework export' - event.info = info - if publish: - event.publish() - for tag in tags: - event.add_tag(tag) - event.distribution = sanitize_misp_event_distribution(distribution) - event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id) - event.analysis = sanitize_misp_event_analysis(analysis) - - misp_objects = ail_objects.get_misp_objects(objs) - for obj in misp_objects: - event.add_object(obj) - - # print(event.to_json()) - - if export: - misp = get_misp_client() - misp_event = misp.add_event(event) - # TODO: handle error - - misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}' - return misp_event - else: - return {'uuid': event['uuid'], 'event': event.to_json()} - -def create_investigation_misp_event(investigation_uuid): - investigation = Investigation(investigation_uuid) - objs = ail_objects.get_objects(investigation.get_objects()) - - event = create_misp_event(objs, - # event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ???? - date=investigation.get_date(), - distribution=0, - threat_level_id=investigation.get_threat_level(), - analysis=investigation.get_analysis(), - info=investigation.get_info(), - tags=investigation.get_tags(), - export=True) - url = event['url'] - if url: - investigation.add_misp_events(url) - return url - -def get_user_misp_objects_to_export(user_id): - objs = [] - objects = r_db.hgetall(f'user:obj:misp:export:{user_id}') - for obj in objects: - obj_type, obj_subtype, obj_id = obj.split(':', 2) - lvl = objects[obj] - try: - lvl = int(lvl) - except: - lvl = 0 - objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl}) - return objs - -def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0): - if not obj_subtype: - obj_subtype = '' - r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl) - -def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id): - r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}') - -def delete_user_misp_objects_to_export(user_id): - r_db.delete(f'user:obj:misp:export:{user_id}') - +#####################################################################3 ########################################################### # # set default diff --git a/bin/exporter/MISPExporter.py b/bin/exporter/MISPExporter.py new file mode 100755 index 00000000..6f7cc3c6 --- /dev/null +++ b/bin/exporter/MISPExporter.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Importer Class +================ + +Import Content + +""" +import os +import sys +import uuid + +from abc import ABC + +from pymisp import MISPEvent, PyMISP +from urllib3 import disable_warnings as urllib3_disable_warnings + +sys.path.append('../../configs/keys') +sys.path.append(os.environ['AIL_BIN']) +################################# +# Import Project packages +################################# +from exporter.abstract_exporter import AbstractExporter +from lib.ConfigLoader import ConfigLoader +from lib.Investigations import Investigation +from lib.objects.abstract_object import AbstractObject + +# from lib.Tracker import Tracker + +config_loader = ConfigLoader() +r_db = config_loader.get_db_conn("Kvrocks_DB") +config_loader = None + +#### FUNCTIONS #### + +def get_user_misp_objects_to_export(user_id): + objs = [] + objects = r_db.hgetall(f'user:obj:misp:export:{user_id}') + for obj in objects: + obj_type, obj_subtype, obj_id = obj.split(':', 2) + lvl = objects[obj] + try: + lvl = int(lvl) + except(TypeError, ValueError): + lvl = 0 + objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl}) + return objs + +def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0): + if not obj_subtype: + obj_subtype = '' + r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl) + +def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id): + r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}') + +def delete_user_misp_objects_to_export(user_id): + r_db.delete(f'user:obj:misp:export:{user_id}') + +# --- FUNCTIONS --- # + +# MISPExporter -> return correct exporter by type ???? +class MISPExporter(AbstractExporter, ABC): # <- AbstractMISPExporter ??????? + """MISP Exporter + + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check or to not check the validity of the certificate. + Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain) + """ + + def __init__(self, url='', key='', ssl=False): + super().__init__() + + if url and key: + self.url = url + self.key = key + self.ssl = ssl + if self.ssl is False: + urllib3_disable_warnings() + elif url or key: + raise Exception('Error: missing url or api key') + else: + try: + from mispKEYS import misp_url, misp_key, misp_verifycert + self.url = misp_url + self.key = misp_key + self.ssl = misp_verifycert + if self.ssl is False: + urllib3_disable_warnings() + if self.url.endswith('/'): + self.url = self.url[:-1] + except Exception: # ModuleNotFoundError + self.url = None + self.key = None + self.ssl = None + + def get_misp(self): + return PyMISP(self.url, self.key, self.ssl) + + # TODO catch exception + def get_misp_uuid(self): + misp = self.get_misp() + misp_setting = misp.get_server_setting('MISP.uuid') + return misp_setting.get('value') + + # TODO ADD TIMEOUT + # TODO return error + def ping_misp(self): + try: + self.get_misp() + return True + except Exception as e: + print(e) + return False + + @staticmethod + def sanitize_distribution(distribution): + try: + int(distribution) + if 0 <= distribution <= 3: + return distribution + else: + return 0 + except (TypeError, ValueError): + return 0 + + @staticmethod + def sanitize_threat_level(threat_level): + try: + int(threat_level) + if 1 <= threat_level <= 4: + return threat_level + else: + return 4 + except (TypeError, ValueError): + return 4 + + @staticmethod + def sanitize_analysis(analysis): + try: + int(analysis) + if 0 <= analysis <= 2: + return analysis + else: + return 0 + except (TypeError, ValueError): + return 0 + + # TODO EVENT REPORT ??????? + def create_event(self, objs, export=False, event_uuid=None, date=None, publish=False, info=None, tags=None, + analysis=0, distribution=0, threat_level=4): + if tags is None: + tags = [] + event = MISPEvent() + if not event_uuid: + event_uuid = str(uuid.uuid4()) + event.uuid = event_uuid + if date: + event.date = date + if not info: + info = 'AIL framework export' + event.info = info + if publish: + event.publish() + for tag in tags: + event.add_tag(tag) + event.distribution = self.sanitize_distribution(distribution) + event.threat_level_id = self.sanitize_threat_level(threat_level) + event.analysis = self.sanitize_analysis(analysis) + + misp_objects = ail_objects.get_misp_objects(objs) + for obj in misp_objects: + event.add_object(obj) + # print(event.to_json()) + + if export: + misp = self.get_misp() + misp_event = misp.add_event(event) + # TODO: handle error + + misp_event['url'] = f'{self.url}/events/view/{misp_event["Event"]["uuid"]}' + return misp_event + else: + return event.to_json() + # return {'uuid': event['uuid'], 'event': event.to_json()} + + # EXPORTER CHAIN + # if self.chainable + # if self.next_exporter: + # next_exporter.export({'type': 'misp_event', 'data': {'event': misp_event}}) + + def __repr__(self): + return f'<{self.__class__.__name__}(url={self.url})' + +class MISPExporterAILObjects(MISPExporter): + """MISPExporter AILObjects + + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain) + """ + + def __init__(self, url='', key='', ssl=False): + super().__init__(url=url, key=key, ssl=ssl) + + def export(self, objects, export=False, event_uuid=None, date=None, publish=False, info=None, tags=[], + analysis=0, distribution=0, threat_level=4): + """Export a list of AILObjects as a MISP event + + :param objects: Investigation object or investigation uuid string + :type objects: list[AbstractObject] + """ + # objects ???? + # TODO convert string tuple to object + + return self.create_event(objects, event_uuid=event_uuid, date=date, publish=publish, + analysis=analysis, distribution=distribution, threat_level=threat_level, + info=info, tags=tags, export=export) + +class MISPExporterInvestigation(MISPExporter): + """MISPExporter Investigation + + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain) + """ + + def __init__(self, url='', key='', ssl=False): + super().__init__(url=url, key=key, ssl=ssl) + + def export(self, investigation): + """Export an Investigation as a MISP event + + :param investigation: Investigation object or investigation uuid string + :type investigation: Investigation | str + """ + if not isinstance(investigation, Investigation): + investigation = Investigation(investigation) + objs = ail_objects.get_objects(investigation.get_objects()) + event = self.create_event(objs, + date=investigation.get_date(), + distribution=0, + threat_level=investigation.get_threat_level(), + analysis=investigation.get_analysis(), + info=investigation.get_info(), + tags=investigation.get_tags(), + export=True) + url = event['url'] + if url: + investigation.add_misp_events(url) + return url + +class MISPExporterTrackerMatch(MISPExporter): + """MISPExporter Tracker match + + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain) + """ + + def __init__(self, url='', key='', ssl=False): + super().__init__(url=url, key=key, ssl=ssl) + + # TODO + def export(self, tracker, item): + pass + + +if __name__ == '__main__': + exporter = MISPExporterAILObjects() + + # from lib.objects.Cves import Cve + # from lib.objects.Items import Item + # objs_t = [Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'), + # Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'), + # Cve('CVE-2015-0410')] + # r = exporter.export(objs_t, export=False) + # print(r) + + r = exporter.get_misp_uuid() + # r = misp.server_settings() + # for item in r['finalSettings']: + # print() + # print(item) + # # print(r['finalSettings'][item]) + # # print() + # print() + + print(r) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py new file mode 100755 index 00000000..97913645 --- /dev/null +++ b/bin/exporter/MailExporter.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Importer Class +================ + +Import Content + +""" +import os +import sys + +from abc import ABC + +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +# from flask import escape + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from exporter.abstract_exporter import AbstractExporter +from lib.ConfigLoader import ConfigLoader +# from lib.objects.abstract_object import AbstractObject +# from lib.Tracker import Tracker + + +class MailExporter(AbstractExporter, ABC): + def __init__(self, host=None, port=None, password=None, user='', sender=''): + super().__init__() + config_loader = ConfigLoader() + + if host: + self.host = host + self.port = port + else: + self.host = config_loader.get_config_str("Notifications", "sender_host") + self.port = config_loader.get_config_int("Notifications", "sender_port") + if password: + self.pw = password + else: + self.pw = config_loader.get_config_str("Notifications", "sender_pw") + if self.pw == 'None': + self.pw = None + if user: + self.user = user + else: + self.user = config_loader.get_config_str("Notifications", "sender_user") + if sender: + self.sender = sender + else: + self.sender = config_loader.get_config_str("Notifications", "sender") + + # raise an exception if any of these is None + if (self.sender is None or + self.host is None or + self.port is None): + raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!') + + def import(self): + pass + + def get_smtp_client(self): + # try: + if self.pw is not None: + try: + smtp_server = smtplib.SMTP(self.host, self.port) + smtp_server.starttls() + except smtplib.SMTPNotSupportedError: + print("The server does not support the STARTTLS extension.") + smtp_server = smtplib.SMTP_SSL(self.host, self.port) + + smtp_server.ehlo() + if self.user is not None: + smtp_server.login(self.user, self.pw) + else: + smtp_server.login(self.sender, self.pw) + else: + smtp_server = smtplib.SMTP(self.host, self.port) + return smtp_server + # except Exception as err: + # traceback.print_tb(err.__traceback__) + # publisher.warning(err) + + def _export(self, recipient, subject, body): + mime_msg = MIMEMultipart() + mime_msg['From'] = self.sender + mime_msg['To'] = recipient + mime_msg['Subject'] = subject + mime_msg.attach(MIMEText(body, 'plain')) + + # try: + smtp_client = self.get_smtp_client() + smtp_client.sendmail(self.sender, recipient, mime_msg.as_string()) + smtp_client.quit() + # except Exception as err: + # traceback.print_tb(err.__traceback__) + # publisher.warning(err) + print(f'Send notification: {subject} to {recipient}') + +class MailExporterTracker(MailExporter): + + def __init__(self, host=None, port=None, password=None, user='', sender=''): + super().__init__(host=host, port=port, password=password, user=user, sender=sender) + + def export(self, tracker, obj): # TODO match + tracker_type = tracker.get_type() + tracker_name = tracker.get_tracker() + subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject + body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" + body += f'Item: {obj.id}\nurl:{obj.get_link()}' + + # TODO match option + # if match: + # body += f'Tracker Match:\n\n{escape(match)}' + + for mail in tracker.get_mails(): + self._export(mail, subject, body) \ No newline at end of file diff --git a/bin/exporter/WebHookExporter.py b/bin/exporter/WebHookExporter.py new file mode 100755 index 00000000..1322a8fa --- /dev/null +++ b/bin/exporter/WebHookExporter.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Importer Class +================ + +Import Content + +""" +import os +import requests +import sys + +from abc import ABC + +sys.path.append(os.environ['AIL_BIN']) +################################# +# Import Project packages +################################# +from exporter.abstract_exporter import AbstractExporter + +# from ConfigLoader import ConfigLoader +# from lib.objects.abstract_object import AbstractObject +# from lib.Tracker import Tracker + +class WebHookExporter(AbstractExporter, ABC): + def __init__(self, url=''): + super().__init__() + self.url = url + + def set_url(self, url): + self.url = url + + def _export(self, data): + try: + response = requests.post(self.url, json=data) + if response.status_code >= 400: + print(f"Webhook request failed for {self.url}\nReason: {response.reason}") + # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except Exception as e: + print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}") + # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + + +class WebHookExporterTracker(WebHookExporter): + + def __init__(self, url=''): + super().__init__(url=url) + + # TODO Change exported keys + def export(self, tracker, obj): + self.set_url(tracker.get_webhook()) + data = {'trackerId': tracker.get_uuid(), + 'trackerType': tracker.get_type(), + 'tags': tracker.get_tags(), + 'tracker': tracker.get_tracker(), + # object + 'itemId': obj.get_id(), + 'itemURL': obj.get_link()} + # Item + # data['itemDate'] = obj.get_date() + # data["itemSource"] = obj.get_source() + + self._export(data) diff --git a/bin/exporter/abstract_exporter.py b/bin/exporter/abstract_exporter.py new file mode 100755 index 00000000..4208673d --- /dev/null +++ b/bin/exporter/abstract_exporter.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Importer Class +================ + +Import Content + +""" +import os +import sys + +from abc import ABC, abstractmethod + + +# sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +# from ConfigLoader import ConfigLoader + +class AbstractExporter(ABC): + def __init__(self): + """ + Init Module + """ + # Module name if provided else instance className + self.name = self._name() + + def _name(self): + """ + Returns the instance class name (ie. the Exporter Name) + """ + return self.__class__.__name__ + + @abstractmethod + def export(self, *args, **kwargs): + """Importer function""" + pass + # res = self.export(*args, **kwargs) + # if self.next_exporter: + # self.next_exporter.exporter(res) + + diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index ecbefade..adc2bcf6 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -27,6 +27,23 @@ r_db = config_loader.get_db_conn('Kvrocks_DB') config_loader = None # --- CONFIG --- # +#### FUNCTIONS #### + +def add_json_feeder_to_queue(json_data): + json_data = json.dumps(json_data) + return r_db.rpush('importer:feeder', json_data) + +def api_add_json_feeder_to_queue(json_data): + if not json_data: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + # # TODO: add JSON verification + res = add_json_feeder_to_queue(json_data) + if not res: + return {'status': 'error'}, 400 + return {'status': 'success'}, 200 + +# --- FUNCTIONS --- # + class FeederImporter(AbstractImporter): def __init__(self): super().__init__() @@ -89,9 +106,8 @@ class FeederModuleImporter(AbstractModule): self.importer = FeederImporter() def get_message(self): - return self.r_db.lpop('importer:feeder') # TODO CHOOSE DB - # TODO RELOAD LIST - # after delta + return self.r_db.lpop('importer:feeder') + # TODO RELOAD LIST after delta def compute(self, message): # TODO HANDLE Invalid JSON @@ -104,20 +120,6 @@ class FeederModuleImporter(AbstractModule): # server_cache.hincrby("mixer_cache:list_feeder", feeder_name, 1) -def add_json_feeder_to_queue(json_data): - json_data = json.dumps(json_data) - return r_db.rpush('importer:feeder', json_data) - -def api_add_json_feeder_to_queue(json_data): - if not json_data: - return {'status': 'error', 'reason': 'Malformed JSON'}, 400 - # # TODO: add JSON verification - res = add_json_feeder_to_queue(json_data) - if not res: - return {'status': 'error'}, 400 - return {'status': 'success'}, 200 - - # Launch Importer if __name__ == '__main__': module = FeederModuleImporter() diff --git a/bin/importer/ZMQImporter.py b/bin/importer/ZMQImporter.py index 67c17d0c..710b3ddc 100755 --- a/bin/importer/ZMQImporter.py +++ b/bin/importer/ZMQImporter.py @@ -9,7 +9,6 @@ Import Content """ import os import sys -import time import zmq @@ -63,6 +62,7 @@ class ZMQModuleImporter(AbstractModule): # TODO register all Importers self.zmq_importer.add(address, channel) + # TODO MESSAGE SOURCE - UI def get_message(self): for message in self.zmq_importer.importer(): # remove channel from message diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 5600b6a4..bab8cd89 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -91,6 +91,9 @@ class Investigation(object): def __init__(self, investigation_uuid): self.uuid = investigation_uuid + def exists(self): + return r_tracking.exists(f'investigations:data:{self.uuid}') + def get_uuid(self, separator=False): if separator: return uuid.UUID(hex=self.uuid, version=4) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 0a939b77..f955be8c 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -11,7 +11,6 @@ It processes every item coming from the global module and test the regex import os import sys import time -import requests sys.path.append(os.environ['AIL_BIN']) ################################## @@ -22,7 +21,8 @@ from lib.objects.Items import Item from packages import Term from lib import Tracker -import NotificationHelper +from exporter.MailExporter import MailExporterTracker +from exporter.WebHookExporter import WebHookExporterTracker class Tracker_Regex(AbstractModule): @@ -38,12 +38,14 @@ class Tracker_Regex(AbstractModule): self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time") - self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" - # refresh Tracked Regex self.dict_regex_tracked = Term.get_regex_tracked_words_dict() self.last_refresh = time.time() + # Exporter + self.exporters = {'mail': MailExporterTracker(), + 'webhook': WebHookExporterTracker()} + self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, item_id): @@ -56,60 +58,45 @@ class Tracker_Regex(AbstractModule): item = Item(item_id) item_id = item.get_id() - item_content = item.get_content() + content = item.get_content() for regex in self.dict_regex_tracked: - matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, item_content) + matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content) if matched: self.new_tracker_found(regex, 'regex', item) - def new_tracker_found(self, tracker, tracker_type, item): - uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type) + # match = self.regex_finditer(self.dict_regex_tracked[regex], item_id, content) + # if match: + # self.new_tracker_found(regex, 'regex', item) + + def new_tracker_found(self, tracker_name, tracker_type, item): + uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) item_id = item.get_id() - item_date = item.get_date() + # date = item.get_date() item_source = item.get_source() - print(f'new tracked regex found: {tracker} in {item_id}') + print(f'new tracked regex found: {tracker_name} in {item_id}') for tracker_uuid in uuid_list: + tracker = Tracker.Tracker(tracker_uuid) + # Source Filtering - tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) + tracker_sources = tracker.get_sources() if tracker_sources and item_source not in tracker_sources: continue - Tracker.add_tracked_item(tracker_uuid, item_id) + Tracker.add_tracked_item(tracker_uuid, item_id) # TODO - tags_to_add = Tracker.get_tracker_tags(tracker_uuid) - for tag in tags_to_add: + for tag in tracker.get_tags(): msg = f'{tag};{item_id}' self.send_message_to_queue(msg, 'Tags') - mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) - mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id) - for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + if tracker.mail_export(): + # TODO add matches + custom subjects + self.exporters['mail'].export(tracker, item) - # Webhook - webhook_to_post = Term.get_term_webhook(tracker_uuid) - if webhook_to_post: - json_request = {"trackerId": tracker_uuid, - "itemId": item_id, - "itemURL": self.full_item_url + item_id, - "tracker": tracker, - "itemSource": item_source, - "itemDate": item_date, - "tags": tags_to_add, - "emailNotification": f'{mail_to_notify}', - "trackerType": tracker_type - } - try: - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") - except: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + if tracker.webhook_export(): + self.exporters['webhook'].export(tracker, item) if __name__ == "__main__": diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index d05c3b22..635c07e0 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Blueprint Flask: MISP format import export -''' +""" import io import os import sys -import uuid import json from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort -from flask_login import login_required, current_user, login_user, logout_user +from flask_login import login_required, current_user sys.path.append('modules') @@ -22,20 +21,22 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from export import Export +from exporter import MISPExporter from lib.objects import ail_objects +from lib.Investigations import Investigation +# TODO REMOVE ME +from export import Export # TODO REMOVE ME from export import MispImport # TODO REMOVE ME - # TODO REMOVE ME # ============ BLUEPRINT ============ import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export')) - # ============ VARIABLES ============ - +misp_exporter_objects = MISPExporter.MISPExporterAILObjects() +misp_exporter_investigation = MISPExporter.MISPExporterInvestigation() # ============ FUNCTIONS ============ @@ -47,6 +48,7 @@ import_export = Blueprint('import_export', __name__, def import_object(): return render_template("import_object.html") + # TODO @import_export.route("/import_export/import_file", methods=['POST']) @login_required @@ -87,7 +89,7 @@ def import_object_file(): def objects_misp_export(): user_id = current_user.get_id() object_types = ail_objects.get_all_objects_with_subtypes_tuple() - to_export = Export.get_user_misp_objects_to_export(user_id) + to_export = MISPExporter.get_user_misp_objects_to_export(user_id) return render_template("export_object.html", object_types=object_types, to_export=to_export) @@ -123,7 +125,7 @@ def objects_misp_export_post(): invalid_obj.append(obj) else: objects.append(obj) - for obj in Export.get_user_misp_objects_to_export(user_id): + for obj in MISPExporter.get_user_misp_objects_to_export(user_id): if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']): invalid_obj.append(obj) else: @@ -139,16 +141,16 @@ def objects_misp_export_post(): export = request.form.get('export_to_misp', False) distribution = request.form.get('misp_event_distribution') - threat_level_id = request.form.get('threat_level_id') + threat_level = request.form.get('threat_level_id') analysis = request.form.get('misp_event_analysis') info = request.form.get('misp_event_info') publish = request.form.get('misp_event_info', False) objs = ail_objects.get_objects(objects) - event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, info=info, export=export, publish=publish) + event = misp_exporter_objects.create_event(objs, distribution=distribution, threat_level=threat_level, + analysis=analysis, info=info, export=export, publish=publish) - Export.delete_user_misp_objects_to_export(user_id) + MISPExporter.delete_user_misp_objects_to_export(user_id) if not export: return send_file(io.BytesIO(event['event'].encode()), as_attachment=True, @@ -176,7 +178,7 @@ def add_object_id_to_export(): if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id): abort(404) - Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl) + MISPExporter.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl) # redirect return redirect(url_for('import_export.objects_misp_export')) @@ -190,7 +192,7 @@ def delete_object_id_to_export(): obj_id = request.args.get('id') obj_subtype = request.args.get('subtype') - Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id) + MISPExporter.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id) return jsonify(success=True) @@ -199,8 +201,11 @@ def delete_object_id_to_export(): @login_analyst def export_investigation(): investigation_uuid = request.args.get("uuid") - if Export.ping_misp(): - event = Export.create_investigation_misp_event(investigation_uuid) + investigation = Investigation(investigation_uuid) + if not investigation.exists(): + abort(404) + if misp_exporter_objects.ping_misp(): + event = misp_exporter_objects.export({'type': 'investigation', 'data': {'investigation': investigation}}) print(event) else: return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), @@ -218,11 +223,12 @@ def create_thehive_case(): tlp = Export.sanitize_tlp_hive(request.form['hive_tlp']) item_id = request.form['obj_id'] - item = Item(item_id) + item = ail_objects.get_object('item', '', item_id) if not item.exists(): abort(404) - case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description) + case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, + description=description) if case_id: return redirect(Export.get_case_url(case_id)) else: