mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 14:17:16 +00:00
chg: [exporter] refactor + add a new way to create and use exporters
This commit is contained in:
Terrtia
parent
de0a60ba8b
commit
24ffcae690
10 changed files with 596 additions and 216 deletions
|
|
@ -199,145 +199,7 @@ def get_item_hive_cases(item_id):
|
|||
# MISP
|
||||
##################################
|
||||
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
|
||||
MISP_URL = misp_url
|
||||
MISP_KEY = misp_key
|
||||
MISP_VERIFYCERT = misp_verifycert
|
||||
if MISP_URL.endswith('/'):
|
||||
MISP_URL = MISP_URL[:-1]
|
||||
except:
|
||||
MISP_URL = None
|
||||
MISP_KEY = None
|
||||
MISP_VERIFYCERT = None
|
||||
|
||||
|
||||
def get_misp_client():
|
||||
return PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
|
||||
|
||||
# # TODO: return error
|
||||
def ping_misp():
|
||||
try:
|
||||
PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
|
||||
def sanitize_misp_event_distribution(distribution):
|
||||
try:
|
||||
int(distribution)
|
||||
if 0 <= distribution <= 3:
|
||||
return distribution
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def sanitize_misp_event_threat_level(threat_level):
|
||||
try:
|
||||
int(threat_level)
|
||||
if 1 <= threat_level <= 4:
|
||||
return threat_level
|
||||
else:
|
||||
return 4
|
||||
except:
|
||||
return 4
|
||||
|
||||
|
||||
def sanitize_misp_event_analysis(analysis):
|
||||
try:
|
||||
int(analysis)
|
||||
if 0 <= analysis <= 2:
|
||||
return analysis
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0,
|
||||
info=None, tags=None, export=False):
|
||||
if tags is None:
|
||||
tags = []
|
||||
event = MISPEvent()
|
||||
if not event_uuid:
|
||||
event_uuid = str(uuid.uuid4())
|
||||
event.uuid = event_uuid
|
||||
if date:
|
||||
event.date = date
|
||||
if not info:
|
||||
info = 'AIL framework export'
|
||||
event.info = info
|
||||
if publish:
|
||||
event.publish()
|
||||
for tag in tags:
|
||||
event.add_tag(tag)
|
||||
event.distribution = sanitize_misp_event_distribution(distribution)
|
||||
event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id)
|
||||
event.analysis = sanitize_misp_event_analysis(analysis)
|
||||
|
||||
misp_objects = ail_objects.get_misp_objects(objs)
|
||||
for obj in misp_objects:
|
||||
event.add_object(obj)
|
||||
|
||||
# print(event.to_json())
|
||||
|
||||
if export:
|
||||
misp = get_misp_client()
|
||||
misp_event = misp.add_event(event)
|
||||
# TODO: handle error
|
||||
|
||||
misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}'
|
||||
return misp_event
|
||||
else:
|
||||
return {'uuid': event['uuid'], 'event': event.to_json()}
|
||||
|
||||
def create_investigation_misp_event(investigation_uuid):
|
||||
investigation = Investigation(investigation_uuid)
|
||||
objs = ail_objects.get_objects(investigation.get_objects())
|
||||
|
||||
event = create_misp_event(objs,
|
||||
# event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ????
|
||||
date=investigation.get_date(),
|
||||
distribution=0,
|
||||
threat_level_id=investigation.get_threat_level(),
|
||||
analysis=investigation.get_analysis(),
|
||||
info=investigation.get_info(),
|
||||
tags=investigation.get_tags(),
|
||||
export=True)
|
||||
url = event['url']
|
||||
if url:
|
||||
investigation.add_misp_events(url)
|
||||
return url
|
||||
|
||||
def get_user_misp_objects_to_export(user_id):
|
||||
objs = []
|
||||
objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
|
||||
for obj in objects:
|
||||
obj_type, obj_subtype, obj_id = obj.split(':', 2)
|
||||
lvl = objects[obj]
|
||||
try:
|
||||
lvl = int(lvl)
|
||||
except:
|
||||
lvl = 0
|
||||
objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
|
||||
return objs
|
||||
|
||||
def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
|
||||
if not obj_subtype:
|
||||
obj_subtype = ''
|
||||
r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
|
||||
|
||||
def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
|
||||
r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
|
||||
|
||||
def delete_user_misp_objects_to_export(user_id):
|
||||
r_db.delete(f'user:obj:misp:export:{user_id}')
|
||||
|
||||
#####################################################################3
|
||||
|
||||
###########################################################
|
||||
# # set default
|
||||
|
|
|
|||
291
bin/exporter/MISPExporter.py
Executable file
291
bin/exporter/MISPExporter.py
Executable file
|
|
@ -0,0 +1,291 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Importer Class
|
||||
================
|
||||
|
||||
Import Content
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from abc import ABC
|
||||
|
||||
from pymisp import MISPEvent, PyMISP
|
||||
from urllib3 import disable_warnings as urllib3_disable_warnings
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
#################################
|
||||
# Import Project packages
|
||||
#################################
|
||||
from exporter.abstract_exporter import AbstractExporter
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.Investigations import Investigation
|
||||
from lib.objects.abstract_object import AbstractObject
|
||||
|
||||
# from lib.Tracker import Tracker
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
r_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
#### FUNCTIONS ####
|
||||
|
||||
def get_user_misp_objects_to_export(user_id):
|
||||
objs = []
|
||||
objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
|
||||
for obj in objects:
|
||||
obj_type, obj_subtype, obj_id = obj.split(':', 2)
|
||||
lvl = objects[obj]
|
||||
try:
|
||||
lvl = int(lvl)
|
||||
except(TypeError, ValueError):
|
||||
lvl = 0
|
||||
objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
|
||||
return objs
|
||||
|
||||
def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
|
||||
if not obj_subtype:
|
||||
obj_subtype = ''
|
||||
r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
|
||||
|
||||
def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
|
||||
r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
|
||||
|
||||
def delete_user_misp_objects_to_export(user_id):
|
||||
r_db.delete(f'user:obj:misp:export:{user_id}')
|
||||
|
||||
# --- FUNCTIONS --- #
|
||||
|
||||
# MISPExporter -> return correct exporter by type ????
|
||||
class MISPExporter(AbstractExporter, ABC): # <- AbstractMISPExporter ???????
|
||||
"""MISP Exporter
|
||||
|
||||
:param url: URL of the MISP instance you want to connect to
|
||||
:param key: API key of the user you want to use
|
||||
:param ssl: can be True or False (to check or to not check the validity of the certificate.
|
||||
Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
|
||||
"""
|
||||
|
||||
def __init__(self, url='', key='', ssl=False):
|
||||
super().__init__()
|
||||
|
||||
if url and key:
|
||||
self.url = url
|
||||
self.key = key
|
||||
self.ssl = ssl
|
||||
if self.ssl is False:
|
||||
urllib3_disable_warnings()
|
||||
elif url or key:
|
||||
raise Exception('Error: missing url or api key')
|
||||
else:
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
self.url = misp_url
|
||||
self.key = misp_key
|
||||
self.ssl = misp_verifycert
|
||||
if self.ssl is False:
|
||||
urllib3_disable_warnings()
|
||||
if self.url.endswith('/'):
|
||||
self.url = self.url[:-1]
|
||||
except Exception: # ModuleNotFoundError
|
||||
self.url = None
|
||||
self.key = None
|
||||
self.ssl = None
|
||||
|
||||
def get_misp(self):
|
||||
return PyMISP(self.url, self.key, self.ssl)
|
||||
|
||||
# TODO catch exception
|
||||
def get_misp_uuid(self):
|
||||
misp = self.get_misp()
|
||||
misp_setting = misp.get_server_setting('MISP.uuid')
|
||||
return misp_setting.get('value')
|
||||
|
||||
# TODO ADD TIMEOUT
|
||||
# TODO return error
|
||||
def ping_misp(self):
|
||||
try:
|
||||
self.get_misp()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def sanitize_distribution(distribution):
|
||||
try:
|
||||
int(distribution)
|
||||
if 0 <= distribution <= 3:
|
||||
return distribution
|
||||
else:
|
||||
return 0
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def sanitize_threat_level(threat_level):
|
||||
try:
|
||||
int(threat_level)
|
||||
if 1 <= threat_level <= 4:
|
||||
return threat_level
|
||||
else:
|
||||
return 4
|
||||
except (TypeError, ValueError):
|
||||
return 4
|
||||
|
||||
@staticmethod
|
||||
def sanitize_analysis(analysis):
|
||||
try:
|
||||
int(analysis)
|
||||
if 0 <= analysis <= 2:
|
||||
return analysis
|
||||
else:
|
||||
return 0
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
# TODO EVENT REPORT ???????
|
||||
def create_event(self, objs, export=False, event_uuid=None, date=None, publish=False, info=None, tags=None,
|
||||
analysis=0, distribution=0, threat_level=4):
|
||||
if tags is None:
|
||||
tags = []
|
||||
event = MISPEvent()
|
||||
if not event_uuid:
|
||||
event_uuid = str(uuid.uuid4())
|
||||
event.uuid = event_uuid
|
||||
if date:
|
||||
event.date = date
|
||||
if not info:
|
||||
info = 'AIL framework export'
|
||||
event.info = info
|
||||
if publish:
|
||||
event.publish()
|
||||
for tag in tags:
|
||||
event.add_tag(tag)
|
||||
event.distribution = self.sanitize_distribution(distribution)
|
||||
event.threat_level_id = self.sanitize_threat_level(threat_level)
|
||||
event.analysis = self.sanitize_analysis(analysis)
|
||||
|
||||
misp_objects = ail_objects.get_misp_objects(objs)
|
||||
for obj in misp_objects:
|
||||
event.add_object(obj)
|
||||
# print(event.to_json())
|
||||
|
||||
if export:
|
||||
misp = self.get_misp()
|
||||
misp_event = misp.add_event(event)
|
||||
# TODO: handle error
|
||||
|
||||
misp_event['url'] = f'{self.url}/events/view/{misp_event["Event"]["uuid"]}'
|
||||
return misp_event
|
||||
else:
|
||||
return event.to_json()
|
||||
# return {'uuid': event['uuid'], 'event': event.to_json()}
|
||||
|
||||
# EXPORTER CHAIN
|
||||
# if self.chainable
|
||||
# if self.next_exporter:
|
||||
# next_exporter.export({'type': 'misp_event', 'data': {'event': misp_event}})
|
||||
|
||||
def __repr__(self):
|
||||
return f'<{self.__class__.__name__}(url={self.url})'
|
||||
|
||||
class MISPExporterAILObjects(MISPExporter):
|
||||
"""MISPExporter AILObjects
|
||||
|
||||
:param url: URL of the MISP instance you want to connect to
|
||||
:param key: API key of the user you want to use
|
||||
:param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
|
||||
"""
|
||||
|
||||
def __init__(self, url='', key='', ssl=False):
|
||||
super().__init__(url=url, key=key, ssl=ssl)
|
||||
|
||||
def export(self, objects, export=False, event_uuid=None, date=None, publish=False, info=None, tags=[],
|
||||
analysis=0, distribution=0, threat_level=4):
|
||||
"""Export a list of AILObjects as a MISP event
|
||||
|
||||
:param objects: Investigation object or investigation uuid string
|
||||
:type objects: list[AbstractObject]
|
||||
"""
|
||||
# objects ????
|
||||
# TODO convert string tuple to object
|
||||
|
||||
return self.create_event(objects, event_uuid=event_uuid, date=date, publish=publish,
|
||||
analysis=analysis, distribution=distribution, threat_level=threat_level,
|
||||
info=info, tags=tags, export=export)
|
||||
|
||||
class MISPExporterInvestigation(MISPExporter):
|
||||
"""MISPExporter Investigation
|
||||
|
||||
:param url: URL of the MISP instance you want to connect to
|
||||
:param key: API key of the user you want to use
|
||||
:param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
|
||||
"""
|
||||
|
||||
def __init__(self, url='', key='', ssl=False):
|
||||
super().__init__(url=url, key=key, ssl=ssl)
|
||||
|
||||
def export(self, investigation):
|
||||
"""Export an Investigation as a MISP event
|
||||
|
||||
:param investigation: Investigation object or investigation uuid string
|
||||
:type investigation: Investigation | str
|
||||
"""
|
||||
if not isinstance(investigation, Investigation):
|
||||
investigation = Investigation(investigation)
|
||||
objs = ail_objects.get_objects(investigation.get_objects())
|
||||
event = self.create_event(objs,
|
||||
date=investigation.get_date(),
|
||||
distribution=0,
|
||||
threat_level=investigation.get_threat_level(),
|
||||
analysis=investigation.get_analysis(),
|
||||
info=investigation.get_info(),
|
||||
tags=investigation.get_tags(),
|
||||
export=True)
|
||||
url = event['url']
|
||||
if url:
|
||||
investigation.add_misp_events(url)
|
||||
return url
|
||||
|
||||
class MISPExporterTrackerMatch(MISPExporter):
|
||||
"""MISPExporter Tracker match
|
||||
|
||||
:param url: URL of the MISP instance you want to connect to
|
||||
:param key: API key of the user you want to use
|
||||
:param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in case of self signed or other certificate (the concatenation of all the crt of the chain)
|
||||
"""
|
||||
|
||||
def __init__(self, url='', key='', ssl=False):
|
||||
super().__init__(url=url, key=key, ssl=ssl)
|
||||
|
||||
# TODO
|
||||
def export(self, tracker, item):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
exporter = MISPExporterAILObjects()
|
||||
|
||||
# from lib.objects.Cves import Cve
|
||||
# from lib.objects.Items import Item
|
||||
# objs_t = [Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
|
||||
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
|
||||
# Cve('CVE-2015-0410')]
|
||||
# r = exporter.export(objs_t, export=False)
|
||||
# print(r)
|
||||
|
||||
r = exporter.get_misp_uuid()
|
||||
# r = misp.server_settings()
|
||||
# for item in r['finalSettings']:
|
||||
# print()
|
||||
# print(item)
|
||||
# # print(r['finalSettings'][item])
|
||||
# # print()
|
||||
# print()
|
||||
|
||||
print(r)
|
||||
121
bin/exporter/MailExporter.py
Executable file
121
bin/exporter/MailExporter.py
Executable file
|
|
@ -0,0 +1,121 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Importer Class
|
||||
================
|
||||
|
||||
Import Content
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
from abc import ABC
|
||||
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
# from flask import escape
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from exporter.abstract_exporter import AbstractExporter
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
# from lib.objects.abstract_object import AbstractObject
|
||||
# from lib.Tracker import Tracker
|
||||
|
||||
|
||||
class MailExporter(AbstractExporter, ABC):
|
||||
def __init__(self, host=None, port=None, password=None, user='', sender=''):
|
||||
super().__init__()
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
if host:
|
||||
self.host = host
|
||||
self.port = port
|
||||
else:
|
||||
self.host = config_loader.get_config_str("Notifications", "sender_host")
|
||||
self.port = config_loader.get_config_int("Notifications", "sender_port")
|
||||
if password:
|
||||
self.pw = password
|
||||
else:
|
||||
self.pw = config_loader.get_config_str("Notifications", "sender_pw")
|
||||
if self.pw == 'None':
|
||||
self.pw = None
|
||||
if user:
|
||||
self.user = user
|
||||
else:
|
||||
self.user = config_loader.get_config_str("Notifications", "sender_user")
|
||||
if sender:
|
||||
self.sender = sender
|
||||
else:
|
||||
self.sender = config_loader.get_config_str("Notifications", "sender")
|
||||
|
||||
# raise an exception if any of these is None
|
||||
if (self.sender is None or
|
||||
self.host is None or
|
||||
self.port is None):
|
||||
raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
|
||||
|
||||
def import(self):
|
||||
pass
|
||||
|
||||
def get_smtp_client(self):
|
||||
# try:
|
||||
if self.pw is not None:
|
||||
try:
|
||||
smtp_server = smtplib.SMTP(self.host, self.port)
|
||||
smtp_server.starttls()
|
||||
except smtplib.SMTPNotSupportedError:
|
||||
print("The server does not support the STARTTLS extension.")
|
||||
smtp_server = smtplib.SMTP_SSL(self.host, self.port)
|
||||
|
||||
smtp_server.ehlo()
|
||||
if self.user is not None:
|
||||
smtp_server.login(self.user, self.pw)
|
||||
else:
|
||||
smtp_server.login(self.sender, self.pw)
|
||||
else:
|
||||
smtp_server = smtplib.SMTP(self.host, self.port)
|
||||
return smtp_server
|
||||
# except Exception as err:
|
||||
# traceback.print_tb(err.__traceback__)
|
||||
# publisher.warning(err)
|
||||
|
||||
def _export(self, recipient, subject, body):
|
||||
mime_msg = MIMEMultipart()
|
||||
mime_msg['From'] = self.sender
|
||||
mime_msg['To'] = recipient
|
||||
mime_msg['Subject'] = subject
|
||||
mime_msg.attach(MIMEText(body, 'plain'))
|
||||
|
||||
# try:
|
||||
smtp_client = self.get_smtp_client()
|
||||
smtp_client.sendmail(self.sender, recipient, mime_msg.as_string())
|
||||
smtp_client.quit()
|
||||
# except Exception as err:
|
||||
# traceback.print_tb(err.__traceback__)
|
||||
# publisher.warning(err)
|
||||
print(f'Send notification: {subject} to {recipient}')
|
||||
|
||||
class MailExporterTracker(MailExporter):
|
||||
|
||||
def __init__(self, host=None, port=None, password=None, user='', sender=''):
|
||||
super().__init__(host=host, port=port, password=password, user=user, sender=sender)
|
||||
|
||||
def export(self, tracker, obj): # TODO match
|
||||
tracker_type = tracker.get_type()
|
||||
tracker_name = tracker.get_tracker()
|
||||
subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject
|
||||
body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
|
||||
body += f'Item: {obj.id}\nurl:{obj.get_link()}'
|
||||
|
||||
# TODO match option
|
||||
# if match:
|
||||
# body += f'Tracker Match:\n\n{escape(match)}'
|
||||
|
||||
for mail in tracker.get_mails():
|
||||
self._export(mail, subject, body)
|
||||
64
bin/exporter/WebHookExporter.py
Executable file
64
bin/exporter/WebHookExporter.py
Executable file
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Importer Class
|
||||
================
|
||||
|
||||
Import Content
|
||||
|
||||
"""
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
|
||||
from abc import ABC
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
#################################
|
||||
# Import Project packages
|
||||
#################################
|
||||
from exporter.abstract_exporter import AbstractExporter
|
||||
|
||||
# from ConfigLoader import ConfigLoader
|
||||
# from lib.objects.abstract_object import AbstractObject
|
||||
# from lib.Tracker import Tracker
|
||||
|
||||
class WebHookExporter(AbstractExporter, ABC):
|
||||
def __init__(self, url=''):
|
||||
super().__init__()
|
||||
self.url = url
|
||||
|
||||
def set_url(self, url):
|
||||
self.url = url
|
||||
|
||||
def _export(self, data):
|
||||
try:
|
||||
response = requests.post(self.url, json=data)
|
||||
if response.status_code >= 400:
|
||||
print(f"Webhook request failed for {self.url}\nReason: {response.reason}")
|
||||
# self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
|
||||
except Exception as e:
|
||||
print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
|
||||
# self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
|
||||
|
||||
|
||||
class WebHookExporterTracker(WebHookExporter):
|
||||
|
||||
def __init__(self, url=''):
|
||||
super().__init__(url=url)
|
||||
|
||||
# TODO Change exported keys
|
||||
def export(self, tracker, obj):
|
||||
self.set_url(tracker.get_webhook())
|
||||
data = {'trackerId': tracker.get_uuid(),
|
||||
'trackerType': tracker.get_type(),
|
||||
'tags': tracker.get_tags(),
|
||||
'tracker': tracker.get_tracker(),
|
||||
# object
|
||||
'itemId': obj.get_id(),
|
||||
'itemURL': obj.get_link()}
|
||||
# Item
|
||||
# data['itemDate'] = obj.get_date()
|
||||
# data["itemSource"] = obj.get_source()
|
||||
|
||||
self._export(data)
|
||||
44
bin/exporter/abstract_exporter.py
Executable file
44
bin/exporter/abstract_exporter.py
Executable file
|
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Importer Class
|
||||
================
|
||||
|
||||
Import Content
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
# from ConfigLoader import ConfigLoader
|
||||
|
||||
class AbstractExporter(ABC):
|
||||
def __init__(self):
|
||||
"""
|
||||
Init Module
|
||||
"""
|
||||
# Module name if provided else instance className
|
||||
self.name = self._name()
|
||||
|
||||
def _name(self):
|
||||
"""
|
||||
Returns the instance class name (ie. the Exporter Name)
|
||||
"""
|
||||
return self.__class__.__name__
|
||||
|
||||
@abstractmethod
|
||||
def export(self, *args, **kwargs):
|
||||
"""Importer function"""
|
||||
pass
|
||||
# res = self.export(*args, **kwargs)
|
||||
# if self.next_exporter:
|
||||
# self.next_exporter.exporter(res)
|
||||
|
||||
|
||||
|
|
@ -27,6 +27,23 @@ r_db = config_loader.get_db_conn('Kvrocks_DB')
|
|||
config_loader = None
|
||||
# --- CONFIG --- #
|
||||
|
||||
#### FUNCTIONS ####
|
||||
|
||||
def add_json_feeder_to_queue(json_data):
|
||||
json_data = json.dumps(json_data)
|
||||
return r_db.rpush('importer:feeder', json_data)
|
||||
|
||||
def api_add_json_feeder_to_queue(json_data):
|
||||
if not json_data:
|
||||
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||
# # TODO: add JSON verification
|
||||
res = add_json_feeder_to_queue(json_data)
|
||||
if not res:
|
||||
return {'status': 'error'}, 400
|
||||
return {'status': 'success'}, 200
|
||||
|
||||
# --- FUNCTIONS --- #
|
||||
|
||||
class FeederImporter(AbstractImporter):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
|
@ -89,9 +106,8 @@ class FeederModuleImporter(AbstractModule):
|
|||
self.importer = FeederImporter()
|
||||
|
||||
def get_message(self):
|
||||
return self.r_db.lpop('importer:feeder') # TODO CHOOSE DB
|
||||
# TODO RELOAD LIST
|
||||
# after delta
|
||||
return self.r_db.lpop('importer:feeder')
|
||||
# TODO RELOAD LIST after delta
|
||||
|
||||
def compute(self, message):
|
||||
# TODO HANDLE Invalid JSON
|
||||
|
|
@ -104,20 +120,6 @@ class FeederModuleImporter(AbstractModule):
|
|||
# server_cache.hincrby("mixer_cache:list_feeder", feeder_name, 1)
|
||||
|
||||
|
||||
def add_json_feeder_to_queue(json_data):
|
||||
json_data = json.dumps(json_data)
|
||||
return r_db.rpush('importer:feeder', json_data)
|
||||
|
||||
def api_add_json_feeder_to_queue(json_data):
|
||||
if not json_data:
|
||||
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||
# # TODO: add JSON verification
|
||||
res = add_json_feeder_to_queue(json_data)
|
||||
if not res:
|
||||
return {'status': 'error'}, 400
|
||||
return {'status': 'success'}, 200
|
||||
|
||||
|
||||
# Launch Importer
|
||||
if __name__ == '__main__':
|
||||
module = FeederModuleImporter()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ Import Content
|
|||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import zmq
|
||||
|
||||
|
|
@ -63,6 +62,7 @@ class ZMQModuleImporter(AbstractModule):
|
|||
# TODO register all Importers
|
||||
self.zmq_importer.add(address, channel)
|
||||
|
||||
# TODO MESSAGE SOURCE - UI
|
||||
def get_message(self):
|
||||
for message in self.zmq_importer.importer():
|
||||
# remove channel from message
|
||||
|
|
|
|||
|
|
@ -91,6 +91,9 @@ class Investigation(object):
|
|||
def __init__(self, investigation_uuid):
|
||||
self.uuid = investigation_uuid
|
||||
|
||||
def exists(self):
|
||||
return r_tracking.exists(f'investigations:data:{self.uuid}')
|
||||
|
||||
def get_uuid(self, separator=False):
|
||||
if separator:
|
||||
return uuid.UUID(hex=self.uuid, version=4)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ It processes every item coming from the global module and test the regex
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
|
|
@ -22,7 +21,8 @@ from lib.objects.Items import Item
|
|||
from packages import Term
|
||||
from lib import Tracker
|
||||
|
||||
import NotificationHelper
|
||||
from exporter.MailExporter import MailExporterTracker
|
||||
from exporter.WebHookExporter import WebHookExporterTracker
|
||||
|
||||
class Tracker_Regex(AbstractModule):
|
||||
|
||||
|
|
@ -38,12 +38,14 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
# refresh Tracked Regex
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
# Exporter
|
||||
self.exporters = {'mail': MailExporterTracker(),
|
||||
'webhook': WebHookExporterTracker()}
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id):
|
||||
|
|
@ -56,60 +58,45 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
item = Item(item_id)
|
||||
item_id = item.get_id()
|
||||
item_content = item.get_content()
|
||||
content = item.get_content()
|
||||
|
||||
for regex in self.dict_regex_tracked:
|
||||
matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, item_content)
|
||||
matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content)
|
||||
if matched:
|
||||
self.new_tracker_found(regex, 'regex', item)
|
||||
|
||||
def new_tracker_found(self, tracker, tracker_type, item):
|
||||
uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type)
|
||||
# match = self.regex_finditer(self.dict_regex_tracked[regex], item_id, content)
|
||||
# if match:
|
||||
# self.new_tracker_found(regex, 'regex', item)
|
||||
|
||||
def new_tracker_found(self, tracker_name, tracker_type, item):
|
||||
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
|
||||
|
||||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
# date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
print(f'new tracked regex found: {tracker} in {item_id}')
|
||||
print(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
|
||||
tracker_sources = tracker.get_sources()
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
continue
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id) # TODO
|
||||
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
for tag in tracker.get_tags():
|
||||
msg = f'{tag};{item_id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
if tracker.mail_export():
|
||||
# TODO add matches + custom subjects
|
||||
self.exporters['mail'].export(tracker, item)
|
||||
|
||||
# Webhook
|
||||
webhook_to_post = Term.get_term_webhook(tracker_uuid)
|
||||
if webhook_to_post:
|
||||
json_request = {"trackerId": tracker_uuid,
|
||||
"itemId": item_id,
|
||||
"itemURL": self.full_item_url + item_id,
|
||||
"tracker": tracker,
|
||||
"itemSource": item_source,
|
||||
"itemDate": item_date,
|
||||
"tags": tags_to_add,
|
||||
"emailNotification": f'{mail_to_notify}',
|
||||
"trackerType": tracker_type
|
||||
}
|
||||
try:
|
||||
response = requests.post(webhook_to_post, json=json_request)
|
||||
if response.status_code >= 400:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
|
||||
except:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
|
||||
if tracker.webhook_export():
|
||||
self.exporters['webhook'].export(tracker, item)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
"""
|
||||
Blueprint Flask: MISP format import export
|
||||
'''
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import json
|
||||
|
||||
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
sys.path.append('modules')
|
||||
|
||||
|
|
@ -22,20 +21,22 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from export import Export
|
||||
from exporter import MISPExporter
|
||||
from lib.objects import ail_objects
|
||||
from lib.Investigations import Investigation
|
||||
|
||||
# TODO REMOVE ME
|
||||
from export import Export # TODO REMOVE ME
|
||||
from export import MispImport # TODO REMOVE ME
|
||||
|
||||
# TODO REMOVE ME
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
import_export = Blueprint('import_export', __name__,
|
||||
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
|
||||
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
misp_exporter_objects = MISPExporter.MISPExporterAILObjects()
|
||||
misp_exporter_investigation = MISPExporter.MISPExporterInvestigation()
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
|
|
@ -47,6 +48,7 @@ import_export = Blueprint('import_export', __name__,
|
|||
def import_object():
|
||||
return render_template("import_object.html")
|
||||
|
||||
|
||||
# TODO
|
||||
@import_export.route("/import_export/import_file", methods=['POST'])
|
||||
@login_required
|
||||
|
|
@ -87,7 +89,7 @@ def import_object_file():
|
|||
def objects_misp_export():
|
||||
user_id = current_user.get_id()
|
||||
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
|
||||
to_export = Export.get_user_misp_objects_to_export(user_id)
|
||||
to_export = MISPExporter.get_user_misp_objects_to_export(user_id)
|
||||
return render_template("export_object.html", object_types=object_types, to_export=to_export)
|
||||
|
||||
|
||||
|
|
@ -123,7 +125,7 @@ def objects_misp_export_post():
|
|||
invalid_obj.append(obj)
|
||||
else:
|
||||
objects.append(obj)
|
||||
for obj in Export.get_user_misp_objects_to_export(user_id):
|
||||
for obj in MISPExporter.get_user_misp_objects_to_export(user_id):
|
||||
if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
|
||||
invalid_obj.append(obj)
|
||||
else:
|
||||
|
|
@ -139,16 +141,16 @@ def objects_misp_export_post():
|
|||
|
||||
export = request.form.get('export_to_misp', False)
|
||||
distribution = request.form.get('misp_event_distribution')
|
||||
threat_level_id = request.form.get('threat_level_id')
|
||||
threat_level = request.form.get('threat_level_id')
|
||||
analysis = request.form.get('misp_event_analysis')
|
||||
info = request.form.get('misp_event_info')
|
||||
publish = request.form.get('misp_event_info', False)
|
||||
|
||||
objs = ail_objects.get_objects(objects)
|
||||
event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id,
|
||||
analysis=analysis, info=info, export=export, publish=publish)
|
||||
event = misp_exporter_objects.create_event(objs, distribution=distribution, threat_level=threat_level,
|
||||
analysis=analysis, info=info, export=export, publish=publish)
|
||||
|
||||
Export.delete_user_misp_objects_to_export(user_id)
|
||||
MISPExporter.delete_user_misp_objects_to_export(user_id)
|
||||
|
||||
if not export:
|
||||
return send_file(io.BytesIO(event['event'].encode()), as_attachment=True,
|
||||
|
|
@ -176,7 +178,7 @@ def add_object_id_to_export():
|
|||
|
||||
if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
|
||||
abort(404)
|
||||
Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
|
||||
MISPExporter.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
|
||||
# redirect
|
||||
return redirect(url_for('import_export.objects_misp_export'))
|
||||
|
||||
|
|
@ -190,7 +192,7 @@ def delete_object_id_to_export():
|
|||
obj_id = request.args.get('id')
|
||||
obj_subtype = request.args.get('subtype')
|
||||
|
||||
Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
|
||||
MISPExporter.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
|
||||
return jsonify(success=True)
|
||||
|
||||
|
||||
|
|
@ -199,8 +201,11 @@ def delete_object_id_to_export():
|
|||
@login_analyst
|
||||
def export_investigation():
|
||||
investigation_uuid = request.args.get("uuid")
|
||||
if Export.ping_misp():
|
||||
event = Export.create_investigation_misp_event(investigation_uuid)
|
||||
investigation = Investigation(investigation_uuid)
|
||||
if not investigation.exists():
|
||||
abort(404)
|
||||
if misp_exporter_objects.ping_misp():
|
||||
event = misp_exporter_objects.export({'type': 'investigation', 'data': {'investigation': investigation}})
|
||||
print(event)
|
||||
else:
|
||||
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
|
||||
|
|
@ -218,11 +223,12 @@ def create_thehive_case():
|
|||
tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
|
||||
item_id = request.form['obj_id']
|
||||
|
||||
item = Item(item_id)
|
||||
item = ail_objects.get_object('item', '', item_id)
|
||||
if not item.exists():
|
||||
abort(404)
|
||||
|
||||
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
|
||||
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level,
|
||||
description=description)
|
||||
if case_id:
|
||||
return redirect(Export.get_case_url(case_id))
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in a new issue