From 5ef29d8abbe977e9a7c6f6d29b75ce4efe1758dc Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 16 May 2023 16:34:22 +0200 Subject: [PATCH] chg: [MISP] refactor MISP thehive auto push --- bin/DB_KVROCKS_MIGRATION.py | 9 +- bin/LAUNCH.sh | 7 +- bin/MISP_The_Hive_feeder.py | 248 ------------ bin/ailleakObject.py | 127 ------- bin/exporter/MISPExporter.py | 84 ++++- bin/exporter/TheHiveExporter.py | 18 +- bin/lib/Tag.py | 85 +++++ bin/modules/IPAddress.py | 2 +- bin/modules/MISP_Thehive_Auto_Push.py | 75 ++++ configs/modules.cfg | 2 +- var/www/blueprints/tags_ui.py | 53 +++ .../modules/PasteSubmit/Flask_PasteSubmit.py | 186 --------- .../templates/edit_tag_export.html | 354 ------------------ var/www/templates/tags/menu_sidebar.html | 2 +- var/www/templates/tags/tags_auto_push.html | 260 +++++++++++++ 15 files changed, 575 insertions(+), 937 deletions(-) delete mode 100755 bin/MISP_The_Hive_feeder.py delete mode 100755 bin/ailleakObject.py create mode 100755 bin/modules/MISP_Thehive_Auto_Push.py delete mode 100644 var/www/modules/PasteSubmit/templates/edit_tag_export.html create mode 100644 var/www/templates/tags/tags_auto_push.html diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index bd6f2ffc..d39fd108 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -292,14 +292,17 @@ def trackers_migration(): print('RETRO HUNT MIGRATION...') for task_uuid in old_Tracker.get_all_retro_hunt_tasks(): - meta = old_Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True, sources=True, tags=True, description=True, nb_match=True) + retro_hunt = Tracker.RetroHunt(retro_hunt) + # TODO Create filters + # TODO GET OLD META + meta = retro_hunt.get_meta(options={'creator', 'date', 'description', 'filter', 'progress', 'tags'}) last_id = old_Tracker.get_retro_hunt_last_analyzed(task_uuid) timeout = old_Tracker.get_retro_hunt_task_timeout(task_uuid) Tracker._re_create_retro_hunt_task(meta['name'], meta['rule'], meta['date'], meta['date_from'], meta['date_to'], meta['creator'], meta['sources'], meta['tags'], [], timeout, meta['description'], task_uuid, state=meta['state'], nb_match=meta['nb_match'], last_id=last_id) # # TODO: IN background ? - for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']): - Tracker.save_retro_hunt_match(task_uuid, id) + for obj_id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']): + retro_hunt.add('item', '', obj_id) Tracker._fix_db_custom_tags() diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 73da0957..85add045 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -268,6 +268,9 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "Zerobins" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Zerobins.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x" + sleep 0.1 + ################################## # TRACKERS MODULES # ################################## @@ -299,9 +302,7 @@ function launching_scripts { ################################## # TO MIGRATE # ################################## - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" - sleep 0.1 -# screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" +# screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" # sleep 0.1 diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py deleted file mode 100755 index 875a2447..00000000 --- a/bin/MISP_The_Hive_feeder.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -""" -module -==================== - -This module send tagged pastes to MISP or THE HIVE Project - -""" -import os -import sys -import uuid -import redis -import time -import json -import binascii -import gzip - -from pubsublogger import publisher -from Helper import Process -import ailleakObject - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Tag - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import ConfigLoader -import item_basic - -from pymisp import PyMISP - -sys.path.append('../configs/keys') - -# import MISP KEYS -try: - from mispKEYS import misp_url, misp_key, misp_verifycert - flag_misp = True -except: - print('Misp keys not present') - flag_misp = False - -# import The Hive Keys -try: - from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert - if the_hive_url == '': - flag_the_hive = False - else: - flag_the_hive = True -except: - print('The HIVE keys not present') - flag_the_hive = False - HiveApi = False - -from thehive4py.api import TheHiveApi -import thehive4py.exceptions -from thehive4py.models import Alert, AlertArtifact -from thehive4py.models import Case, CaseTask, CustomFieldHelper - -def is_gzip_file(magic_nuber): - return binascii.hexlify(magic_nuber) == b'1f8b' - -def create_the_hive_alert(source, item_id, tag): - # # TODO: check items status (processed by all modules) - # # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation... - # # # TODO: description, add AIL link:show items ? - tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) ) - - path = item_basic.get_item_filepath(item_id) - paste_handle = open(path, 'rb') - paste_data = paste_handle.read() - tmp_path = None - - if is_gzip_file(paste_data[0:2]): # if gzip, create a new file to supply to TheHive - paste_handle.close() # TheHive expects a file handle, that's why we create a new file - tmp_data = gzip.decompress(paste_data) - tmp_path = path + '.unzip' - with open(tmp_path, 'wb+') as f: - f.write(tmp_data) - paste_handle = open(tmp_path, 'rb') - if path.endswith(".gz"): # remove .gz from submitted path to TheHive beause we've decompressed it - path = path[:-3] - - path = os.path.basename(os.path.normpath(path)) + ".txt" # get last part of path, add .txt so it's easier to open when downloaded from TheHive - - artifacts = [ - AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), - AlertArtifact( dataType='file', data=(paste_handle, path), tags=tags ) - ] - - # Prepare the sample Alert - sourceRef = str(uuid.uuid4())[0:6] - alert = Alert(title='AIL Leak', - tlp=3, - tags=tags, - description='AIL Leak, triggered by {}'.format(tag), - type='ail', - source=source, - sourceRef=sourceRef, - artifacts=artifacts) - - # Create the Alert - id = None - try: - response = HiveApi.create_alert(alert) - if response.status_code == 201: - #print(json.dumps(response.json(), indent=4, sort_keys=True)) - print('Alert Created') - print('') - id = response.json()['id'] - else: - print('ko: {}/{}'.format(response.status_code, response.text)) - return 0 - except: - print('hive connection error') - - paste_handle.close() - if tmp_path is not None: # this file has been send to TheHive, we won't ever need it again - os.remove(tmp_path) - -def feeder(message, count=0): - - if flag_the_hive or flag_misp: - tag, item_id = message.split(';') - - ## FIXME: remove it - if not item_basic.exist_item(item_id): - if count < 10: - r_serv_db.zincrby('mess_not_saved_export', 1, message) - return 0 - else: - r_serv_db.zrem('mess_not_saved_export', message) - print('Error: {} do not exist, tag= {}'.format(item_id, tag)) - return 0 - - source = item_basic.get_source(item_id) - - if HiveApi != False: - if int(r_serv_db.get('hive:auto-alerts')) == 1: - if r_serv_db.sismember('whitelist_hive', tag): - create_the_hive_alert(source, item_id, tag) - else: - print('hive, auto alerts creation disable') - if flag_misp: - if int(r_serv_db.get('misp:auto-events')) == 1: - if r_serv_db.sismember('whitelist_misp', tag): - misp_wrapper.pushToMISP(uuid_ail, item_id, tag) - else: - print('misp, auto events creation disable') - - -if __name__ == "__main__": - - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'MISP_The_hive_feeder' - - config_loader = ConfigLoader.ConfigLoader() - - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - - # set sensor uuid - uuid_ail = r_serv_db.get('ail:uuid') - if uuid_ail is None: - uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() ) - - # set default - if r_serv_db.get('hive:auto-alerts') is None: - r_serv_db.set('hive:auto-alerts', 0) - - if r_serv_db.get('misp:auto-events') is None: - r_serv_db.set('misp:auto-events', 0) - - p = Process(config_section) - # create MISP connection - if flag_misp: - try: - pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - except: - flag_misp = False - r_serv_db.set('ail:misp', False) - print('Not connected to MISP') - - if flag_misp: - #try: - misp_wrapper = ailleakObject.ObjectWrapper(pymisp) - r_serv_db.set('ail:misp', True) - print('Connected to MISP:', misp_url) - #except Exception as e: - # flag_misp = False - # r_serv_db.set('ail:misp', False) - # print(e) - # print('Not connected to MISP') - - # create The HIVE connection - if flag_the_hive: - try: - HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert = the_hive_verifycert) - except: - HiveApi = False - flag_the_hive = False - r_serv_db.set('ail:thehive', False) - print('Not connected to The HIVE') - else: - HiveApi = False - - if HiveApi != False and flag_the_hive: - try: - HiveApi.get_alert(0) - r_serv_db.set('ail:thehive', True) - print('Connected to The HIVE:', the_hive_url) - except thehive4py.exceptions.AlertException: - HiveApi = False - flag_the_hive = False - r_serv_db.set('ail:thehive', False) - print('Not connected to The HIVE') - - refresh_time = 3 - ## FIXME: remove it - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) - config_loader = None - - time_1 = time.time() - - while True: - - # Get one message from the input queue - message = p.get_from_set() - if message is None: - - # handle not saved pastes - if int(time.time() - time_1) > refresh_time: - - num_queu = r_serv_db.zcard('mess_not_saved_export') - list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True) - - if num_queu and list_queu: - for i in range(0, num_queu): - feeder(list_queu[i][0],list_queu[i][1]) - - time_1 = time.time() - else: - publisher.debug("{} queue is empty, waiting 1s".format(config_section)) - time.sleep(1) - else: - feeder(message) diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py deleted file mode 100755 index ca4efb43..00000000 --- a/bin/ailleakObject.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -from pymisp import MISPEvent, MISPObject -from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator - -import datetime - -from lib.objects.Items import Item -from lib import ConfigLoader - -class ObjectWrapper: - def __init__(self, pymisp): - self.pymisp = pymisp - self.currentID_date = None - self.eventID_to_push = self.get_daily_event_id() - config_loader = ConfigLoader.ConfigLoader() - self.maxDuplicateToPushToMISP = config_loader.get_config_int("ailleakObject", "maxDuplicateToPushToMISP") - config_loader = None - self.attribute_to_tag = None - - def add_new_object(self, uuid_ail, item_id): - self.uuid_ail = uuid_ail - - item = Item(item_id) - return item.get_misp_object() - - def date_to_str(self, date): - return "{0}-{1}-{2}".format(date.year, date.month, date.day) - - def get_all_related_events(self, to_search): - result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False) - events = [] - if result: - for e in result: - events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) - return events - - def get_daily_event_id(self): - to_match = "Daily AIL-leaks {}".format(datetime.date.today()) - events = self.get_all_related_events(to_match) - for dic in events: - info = dic['info'] - e_id = dic['id'] - if info == to_match: - print('Found: ', info, '->', e_id) - self.currentID_date = datetime.date.today() - return e_id - created_event = self.create_daily_event() - new_id = created_event['Event']['id'] - print('New event created:', new_id) - self.currentID_date = datetime.date.today() - return new_id - - - def create_daily_event(self): - today = datetime.date.today() - # [0-3] - distribution = 0 - info = "Daily AIL-leaks {}".format(today) - # [0-2] - analysis = 0 - # [1-4] - threat = 3 - published = False - org_id = None - orgc_id = None - sharing_group_id = None - date = None - - event = MISPEvent() - event.distribution = distribution - event.info = info - event.analysis = analysis - event.threat = threat - event.published = published - - event.add_tag('infoleak:output-format="ail-daily"') - existing_event = self.pymisp.add_event(event) - return existing_event - - # Publish object to MISP - def pushToMISP(self, uuid_ail, item_id, tag): - - if self.currentID_date != datetime.date.today(): #refresh id - self.eventID_to_push = self.get_daily_event_id() - - mispTYPE = 'ail-leak' - - # paste object already exist - if self.paste_object_exist(self.eventID_to_push, item_id): - # add new tag - self.tag(self.attribute_to_tag, tag) - print(item_id + ' tagged: ' + tag) - # create object - else: - misp_obj = self.add_new_object(uuid_ail, item_id) - - # deprecated - # try: - # templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] - # except IndexError: - # valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) - # print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - - - r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True) - if 'errors' in r: - print(r) - else: - print('Pushed:', tag, '->', item_id) - - def paste_object_exist(self, eventId, item_id): - res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id) - # object already exist - if res.get('Attribute', []): - self.attribute_to_tag = res['Attribute'][0]['uuid'] - return True - # new object - else: - return False - - def tag(self, uuid, tag): - self.pymisp.tag(uuid, tag) diff --git a/bin/exporter/MISPExporter.py b/bin/exporter/MISPExporter.py index f07382e0..4a0d3a1c 100755 --- a/bin/exporter/MISPExporter.py +++ b/bin/exporter/MISPExporter.py @@ -8,6 +8,7 @@ Import Content """ import os +import datetime import sys import uuid @@ -154,6 +155,36 @@ class MISPExporter(AbstractExporter, ABC): except (TypeError, ValueError): return 0 + def get_event_object_id(self, event_id, obj): + misp = self.get_misp() + resp = misp.search(controller='attributes', eventid=event_id, value=obj.get_id()) + attribute = resp.get('Attribute', []) + if attribute: + return attribute[0]['object_id'] + + def add_event_object_tag(self, obj_id, tag): + misp = self.get_misp() + misp_obj = misp.get_object(obj_id, pythonify=True) + for attribute in misp_obj.attributes: + attribute.add_tag(tag) + misp.update_attribute(attribute) + + def add_event_object(self, event_id, obj): + misp_object = obj.get_misp_object() + misp = self.get_misp() + misp.add_object(event_id, misp_object) + + def get_daily_event_id(self): + misp = self.get_misp() + event_info = f'Daily AIL-leaks {datetime.date.today()}' + resp = misp.search(controller='events', eventinfo=event_info, metadata=True) + if resp: + return resp[0]['Event']['id'] + else: + misp_event = self.create_event([], info=event_info, threat_level=3, export=True) + return misp_event['Event']['id'] + + # TODO EVENT REPORT ??????? def create_event(self, objs, export=False, event_uuid=None, date=None, publish=False, info=None, tags=None, analysis=0, distribution=0, threat_level=4): @@ -277,18 +308,57 @@ class MISPExporterTrackerMatch(MISPExporter): pass -if __name__ == '__main__': - exporter = MISPExporterAILObjects() +class MISPExporterAutoDaily(MISPExporter): + """MISPExporter AILObjects + :param url: URL of the MISP instance you want to connect to :param key: API key of the user you want to use + :param ssl: can be True or False (to check or to not check the validity of the certificate. Or a CA_BUNDLE in + case of self signed or other certificate (the concatenation of all the crt of the chain) + """ + + def __init__(self, url='', key='', ssl=False): + super().__init__(url=url, key=key, ssl=ssl) + + # create event if don't exists + try: + self.event_id = self.get_daily_event_id() + except MISPConnectionError: + self.event_id = - 1 + self.date = datetime.date.today() + + def export(self, obj, tag): + """Export a list of AILObjects as a MISP event + + :param obj: AIL Object to export + :type obj: AbstractObject + """ + try: + if self.date != datetime.date.today() or int(self.event_id) < 0: + self.date = datetime.date.today() + self.event_id = self.get_daily_event_id() + + obj_id = self.get_event_object_id(self.event_id, obj) + # Object already in event + if obj_id: + self.add_event_object_tag(obj_id, tag) + else: + self.add_event_object(self.event_id, obj) + + except MISPConnectionError: + return -1 + + +if __name__ == '__main__': + # exporter = MISPExporterAILObjects() # from lib.objects.Cves import Cve - # from lib.objects.Items import Item + from lib.objects.Items import Item # objs_t = [Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'), # Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'), # Cve('CVE-2015-0410')] # r = exporter.export(objs_t, export=False) # print(r) - r = exporter.get_misp_uuid() + # r = exporter.get_misp_uuid() # r = misp.server_settings() # for item in r['finalSettings']: # print() @@ -297,4 +367,8 @@ if __name__ == '__main__': # # print() # print() - print(r) + obj = Item('submitted/2023/05/15/submitted_aed90c6f-c620-4437-93d7-5ff17d1a8eef.gz') + obj = Item('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz') + tag = 'infoleak:automatic-detection="credit-card"' + exporter = MISPExporterAutoDaily() + exporter.export(obj, tag) diff --git a/bin/exporter/TheHiveExporter.py b/bin/exporter/TheHiveExporter.py index e943861c..5a8a2eab 100755 --- a/bin/exporter/TheHiveExporter.py +++ b/bin/exporter/TheHiveExporter.py @@ -143,14 +143,16 @@ class TheHiveExporter(AbstractExporter, ABC): if req.status_code == 201: # print(json.dumps(req.json(), indent=4, sort_keys=True)) print('Alert Created') - print(req.json()) + # print(req.json()) alert_id = req.json()['id'] else: # TODO LOGS print(f'ko: {req.status_code}/{req.text}') - return 0 - except: + return -2 + except Exception as e: + print(e) print('hive connection error') + return -1 return alert_id def create_case(self, observables, description=None, tags=None, title=None, threat_level=2, tlp=2): @@ -187,8 +189,8 @@ class TheHiveExporter(AbstractExporter, ABC): return f'<{self.__class__.__name__}(url={self.url})' -class TheHiveExporterTagTrigger(TheHiveExporter): - """TheHiveExporter TagTrigger +class TheHiveExporterAlertTag(TheHiveExporter): + """TheHiveExporterAlertTag TagTrigger :param url: URL of the Hive instance you want to connect to :param key: API key of the user you want to use @@ -198,9 +200,9 @@ class TheHiveExporterTagTrigger(TheHiveExporter): def __init__(self, url='', key='', ssl=False): super().__init__(url=url, key=key, ssl=ssl) - def export(self, item_id, tag): - item = Item(item_id) - tags = item.get_tags() + def export(self, item, tag): + item_id = item.get_id() + tags = list(item.get_tags()) # remove .gz from submitted path to TheHive because content is decompressed if item_id.endswith(".gz"): diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index a3a5f962..619dde67 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* +import time import redis import datetime @@ -24,6 +25,7 @@ from pymispgalaxies import Galaxies, Clusters config_loader = ConfigLoader() r_tags = config_loader.get_db_conn("Kvrocks_Tags") +r_cache = config_loader.get_redis_conn("Redis_Cache") config_loader = None #### CORE FUNCTIONS #### @@ -1240,6 +1242,89 @@ class Tag: return meta +#### TAG AUTO PUSH #### + +def get_auto_push_status(): + meta = {} + for name in ['misp', 'thehive']: + meta[name] = r_cache.hget('auto:push:status', name) + return meta + +def set_auto_push_status(name, status): + return r_cache.hset('auto:push:status', name, status) + +def get_last_auto_push_refreshed(): + last = r_cache.get('auto:push:refreshed') + if not last: + return -1 + else: + return int(last) + +def _set_last_auto_push_refreshed(): + return r_cache.set('auto:push:refreshed', int(time.time())) + +def is_auto_push_enabled(name): + enabled = r_tags.hget('auto:push', name) + if enabled: + return int(enabled) == 1 + else: + disable_auto_push(name) + return False + +def enable_auto_push(name): + r_tags.hset('auto:push', name, 1) + +def disable_auto_push(name): + r_tags.hset('auto:push', name, 0) + +def get_auto_push_enabled_tags(name): + return r_tags.smembers(f'auto:push:tags:{name}') + +def _add_auto_push_enabled_tags(name, tag): + return r_tags.sadd(f'auto:push:tags:{name}', tag) + +def _del_auto_push_enabled_tags(name): + return r_tags.delete(f'auto:push:tags:{name}') + +def api_add_auto_push_enabled_tags(data): + misp_tags = data.get('misp_tags', []) + thehive_tags = data.get('thehive_tags', []) + for tag in misp_tags: + if not is_taxonomie_tag(tag, 'infoleak') and not is_custom_tag(tag): + return {'error': f'Invalid Tag: {tag}'}, 400 + for tag in thehive_tags: + if not is_taxonomie_tag(tag, 'infoleak') and not is_custom_tag(tag): + return {'error': f'Invalid Tag: {tag}'}, 400 + + _del_auto_push_enabled_tags('misp') + for tag in misp_tags: + _add_auto_push_enabled_tags('misp', tag) + _del_auto_push_enabled_tags('thehive') + for tag in thehive_tags: + _add_auto_push_enabled_tags('thehive', tag) + +def get_auto_push_tags(): + tags = get_taxonomie_enabled_tags('infoleak', r_list=True) + tags[0:0] = list(get_all_custom_tags()) + return tags + +def get_auto_push_enabled_meta(): + meta = {} + for name in ['misp', 'thehive']: + meta[name] = {'enabled': is_auto_push_enabled(name)} + meta[name]['tags'] = get_auto_push_enabled_tags(name) + return meta + +def refresh_auto_push(): + meta = {} + for name in ['misp', 'thehive']: + if is_auto_push_enabled(name): + meta[name] = get_auto_push_enabled_tags(name) + _set_last_auto_push_refreshed() + return meta + +# --- TAG AUTO PUSH --- # + ################################################################################### ################################################################################### ################################################################################### diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 8f4ea65f..5f86bf90 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -31,7 +31,7 @@ from lib import regex_helper # TODO REWRITE ME -> PERF + IPV6 + Tracker ? class IPAddress(AbstractModule): - """Telegram module for AIL framework""" + """IPAddress module for AIL framework""" def __init__(self): super(IPAddress, self).__init__() diff --git a/bin/modules/MISP_Thehive_Auto_Push.py b/bin/modules/MISP_Thehive_Auto_Push.py new file mode 100755 index 00000000..3c7fc80f --- /dev/null +++ b/bin/modules/MISP_Thehive_Auto_Push.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +module +==================== + +This module send tagged pastes to MISP or THE HIVE Project + +""" +import os +import sys +import time + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from lib.exceptions import MISPConnectionError +from lib.objects.Items import Item +from lib import Tag +from exporter.MISPExporter import MISPExporterAutoDaily +from exporter.TheHiveExporter import TheHiveExporterAlertTag + +class MISP_Thehive_Auto_Push(AbstractModule): + """MISP_Hive_Feeder module for AIL framework""" + + def __init__(self): + super(MISP_Thehive_Auto_Push, self).__init__() + + # refresh Tracked Regex + self.tags = Tag.refresh_auto_push() + self.last_refresh = time.time() + + self.misp_exporter = MISPExporterAutoDaily() + self.the_hive_exporter = TheHiveExporterAlertTag() + + # Send module state to logs + self.logger.info(f"Module {self.module_name} initialized") + + def compute(self, message): + if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0: + self.tags = Tag.refresh_auto_push() + self.last_refresh = time.time() + self.redis_logger.info('Tags Auto Push refreshed') + + item_id, tag = message.split(' ', 1) + item = Item(item_id) + + # enabled + if 'misp' in self.tags: + if tag in self.tags['misp']: + r = self.misp_exporter.export(item, tag) + if r == -1: + Tag.set_auto_push_status('misp', 'ConnectionError') + else: + Tag.set_auto_push_status('misp', '') + self.logger.info('MISP Pushed:', tag, '->', item_id) + + if 'thehive' in self.tags: + if tag in self.tags['thehive']: + r = self.the_hive_exporter.export(item, tag) + if r == -1: + Tag.set_auto_push_status('thehive', 'ConnectionError') + elif r == -2: + Tag.set_auto_push_status('thehive', 'Request Entity Too Large') + else: + Tag.set_auto_push_status('thehive', '') + self.logger.info('thehive Pushed:', tag, '->', item_id) + + +if __name__ == "__main__": + module = MISP_Thehive_Auto_Push() + module.run() diff --git a/configs/modules.cfg b/configs/modules.cfg index f5b751f5..360dfbf1 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -111,7 +111,7 @@ publish = Tag_feed,Sync [Sync_module] subscribe = Sync -[MISP_The_hive_feeder] +[MISP_Thehive_Auto_Push] subscribe = Tag_feed #[SentimentAnalysis] diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index f7451afb..7199b98c 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -366,3 +366,56 @@ def get_obj_by_tags(): dict_tagged['date'] = dict_obj['date'] return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + + +@tags_ui.route("/tags/auto_push") +@login_required +@login_analyst +def auto_push(): + + # TODO CHECK if misp or the hive connected + + meta = Tag.get_auto_push_enabled_meta() + auto_push_tags = Tag.get_auto_push_tags() + return render_template("tags_auto_push.html", + auto_push_tags=auto_push_tags, + meta=meta, + auto_push_status=Tag.get_auto_push_status()) + +@tags_ui.route("/tags/auto_push_post", methods=['POST']) +@login_required +@login_analyst +def auto_push_post(): + tag_enabled_misp = request.form.getlist('tag_enabled_misp') + tag_enabled_hive = request.form.getlist('tag_enabled_hive') + + Tag.api_add_auto_push_enabled_tags({'misp_tags': tag_enabled_misp, 'thehive_tags': tag_enabled_hive}) + return redirect(url_for('tags_ui.auto_push')) + +@tags_ui.route("/tags/auto_push/misp/enable") +@login_required +@login_analyst +def enable_misp_auto_push(): + Tag.enable_auto_push('misp') + return redirect(url_for('tags_ui.auto_push')) + +@tags_ui.route("/tags/auto_push/misp/disable") +@login_required +@login_analyst +def disable_misp_auto_push(): + Tag.disable_auto_push('misp') + return redirect(url_for('tags_ui.auto_push')) + +@tags_ui.route("/tags/auto_push/thehive/enable") +@login_required +@login_analyst +def enable_hive_auto_push(): + Tag.enable_auto_push('thehive') + return redirect(url_for('tags_ui.auto_push')) + +@tags_ui.route("/tags/auto_push/thehive/disable") +@login_required +@login_analyst +def disable_hive_auto_push(): + Tag.disable_auto_push('thehive') + return redirect(url_for('tags_ui.auto_push')) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 7d918482..5adc59b9 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -29,8 +29,6 @@ from lib import Tag from packages import Import_helper -from pytaxonomies import Taxonomies # TODO REMOVE ME - # ============ VARIABLES ============ import Flask_config @@ -299,189 +297,5 @@ def submit_status(): return 'INVALID UUID' -###################################################################################################### -###################################################################################################### -###################################################################################################### -###################################################################################################### - -# TODO MIGRATE TAGS PUSH - -# TEMPORARY: 2 SET OF CUSTOM + infoleak tags ????????? - - -@PasteSubmit.route("/PasteSubmit/edit_tag_export") -@login_required -@login_analyst -def edit_tag_export(): - return abort(404) - - misp_auto_events = r_serv_db.get('misp:auto-events') - hive_auto_alerts = r_serv_db.get('hive:auto-alerts') - - whitelist_misp = r_serv_db.scard('whitelist_misp') - whitelist_hive = r_serv_db.scard('whitelist_hive') - - list_export_tags = list(r_serv_db.smembers('list_export_tags')) - status_misp = [] - status_hive = [] - - infoleak_tags = Taxonomies().get('infoleak').machinetags() - is_infoleak_tag = [] - - for tag in list_export_tags: - if r_serv_db.sismember('whitelist_misp', tag): - status_misp.append(True) - else: - status_misp.append(False) - - for tag in list_export_tags: - if r_serv_db.sismember('whitelist_hive', tag): - status_hive.append(True) - else: - status_hive.append(False) - - if tag in infoleak_tags: - is_infoleak_tag.append(True) - else: - is_infoleak_tag.append(False) - - if misp_auto_events is not None: - if int(misp_auto_events) == 1: - misp_active = True - else: - misp_active = False - else: - misp_active = False - - if hive_auto_alerts is not None: - if int(hive_auto_alerts) == 1: - hive_active = True - else: - hive_active = False - else: - hive_active = False - - nb_tags = str(r_serv_db.scard('list_export_tags')) - nb_tags_whitelist_misp = str(r_serv_db.scard('whitelist_misp')) + ' / ' + nb_tags - nb_tags_whitelist_hive = str(r_serv_db.scard('whitelist_hive')) + ' / ' + nb_tags - - return render_template("edit_tag_export.html", - misp_active=misp_active, - hive_active=hive_active, - list_export_tags=list_export_tags, - is_infoleak_tag=is_infoleak_tag, - status_misp=status_misp, - status_hive=status_hive, - nb_tags_whitelist_misp=nb_tags_whitelist_misp, - nb_tags_whitelist_hive=nb_tags_whitelist_hive, - flag_misp=True, - flag_hive=True) - -@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST']) -@login_required -@login_analyst -def tag_export_edited(): - return abort(404) - - - tag_enabled_misp = request.form.getlist('tag_enabled_misp') - tag_enabled_hive = request.form.getlist('tag_enabled_hive') - - list_export_tags = list(r_serv_db.smembers('list_export_tags')) - - r_serv_db.delete('whitelist_misp') - r_serv_db.delete('whitelist_hive') - - for tag in tag_enabled_misp: - if r_serv_db.sismember('list_export_tags', tag): - r_serv_db.sadd('whitelist_misp', tag) - else: - return 'invalid input' - - for tag in tag_enabled_hive: - if r_serv_db.sismember('list_export_tags', tag): - r_serv_db.sadd('whitelist_hive', tag) - else: - return 'invalid input' - - return redirect(url_for('PasteSubmit.edit_tag_export')) - -@PasteSubmit.route("/PasteSubmit/enable_misp_auto_event") -@login_required -@login_analyst -def enable_misp_auto_event(): - return abort(404) - - r_serv_db.set('misp:auto-events', 1) - return edit_tag_export() - -@PasteSubmit.route("/PasteSubmit/disable_misp_auto_event") -@login_required -@login_analyst -def disable_misp_auto_event(): - return abort(404) - - r_serv_db.set('misp:auto-events', 0) - return edit_tag_export() - -@PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert") -@login_required -@login_analyst -def enable_hive_auto_alert(): - return abort(404) - - r_serv_db.set('hive:auto-alerts', 1) - return edit_tag_export() - -@PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert") -@login_required -@login_analyst -def disable_hive_auto_alert(): - return abort(404) - - r_serv_db.set('hive:auto-alerts', 0) - return edit_tag_export() - -@PasteSubmit.route("/PasteSubmit/add_push_tag") -@login_required -@login_analyst -def add_push_tag(): - return abort(404) - - tag = request.args.get('tag') - if tag is not None: - - #limit tag length - if len(tag) > 49: - tag = tag[0:48] - - r_serv_db.sadd('list_export_tags', tag) - - to_return = {} - to_return["tag"] = tag - return jsonify(to_return) - else: - return 'None args', 400 - -@PasteSubmit.route("/PasteSubmit/delete_push_tag") -@login_required -@login_analyst -def delete_push_tag(): - return abort(404) - - - tag = request.args.get('tag') - - infoleak_tags = Taxonomies().get('infoleak').machinetags() - if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): - r_serv_db.srem('list_export_tags', tag) - r_serv_db.srem('whitelist_misp', tag) - r_serv_db.srem('whitelist_hive', tag) - to_return = {} - to_return["tag"] = tag - return jsonify(to_return) - else: - return 'this tag can\'t be removed', 400 - # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit, url_prefix=baseUrl) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html deleted file mode 100644 index 45f56e5c..00000000 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ /dev/null @@ -1,354 +0,0 @@ - - - - - - - - Tags Export - AIL - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
-
-
- -
- - -
-
-
- -
-
MISP Auto Event Creation - {% if misp_active %} - Enabled - {% endif %} - {% if not misp_active %} - Disabled - {% endif %} -
-
- -
- -
-
- - {% if flag_misp %} - {% if misp_active %} - - Disable Event Creation - - {% endif %} - {% if not misp_active %} - - Enable Event Creation - - {% endif %} - {% endif %} - - {% if not flag_misp %} - - {% endif %} -
-
-
- -
-
-
The hive auto export - {% if hive_active %} - Enabled - {% endif %} - {% if not hive_active %} - Disabled - {% endif %} -
-
- -
- -
- - {% if flag_hive %} - {% if hive_active %} - - Disable Alert Creation - - {% endif %} - {% if not hive_active %} - - Enable Alert Creation - - {% endif %} - {% endif %} - {% if not flag_hive %} - - {% endif %} -
-
-
- -
-
- -
- - -
-
- -
- -
-
- Metadata : -     - {{ nb_tags_whitelist_misp }} -
  • - - - - - - - - - - {% for tag in list_export_tags %} - - - - - {% endfor %} - - -
    WhitelistTag
    - {% if status_misp[loop.index0] %} -
    Enabled
    - - {% endif %} - {% if not status_misp[loop.index0] %} -
    Disabled
    - - {% endif %} -     		- {{ tag }} - {% if not is_infoleak_tag[loop.index0] %} -
    - -
    - {% endif %} -
    -
-
-
- -
- -
-
- Metadata : -     - {{ nb_tags_whitelist_hive }} -
  • - - - - - - - - - - - {% for tag in list_export_tags %} - - - - - {% endfor %} - - -
    WhitelistTag
    - {% if status_hive[loop.index0] %} -
    Enabled
    - - {% endif %} - {% if not status_hive[loop.index0] %} -
    Disabled
    - - {% endif %} -     		- {{ tag }} - {% if not is_infoleak_tag[loop.index0] %} -
    - -
    - {% endif %} -
    -
-
-
- -
-
- - -
- -
- -
- -
- - - - -
- -
- - - - - - - - - - diff --git a/var/www/templates/tags/menu_sidebar.html b/var/www/templates/tags/menu_sidebar.html index 6a8cec37..52d4f507 100644 --- a/var/www/templates/tags/menu_sidebar.html +++ b/var/www/templates/tags/menu_sidebar.html @@ -57,7 +57,7 @@