From 4d0e9a94d0a62d7d8d26a88838023dd5dcbd539a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 17 Feb 2020 17:01:16 +0100 Subject: [PATCH] chg: [UI MISP export] add MISP export --- bin/export/MispExport.py | 38 ++++++--- bin/lib/Correlate_object.py | 17 +++- bin/packages/Correlation.py | 6 ++ var/www/blueprints/import_export.py | 44 +++++++++- var/www/templates/decoded/menu_sidebar.html | 4 +- .../import_export/export_object.html | 80 ++++++++++++++++++- .../import_export/import_object.html | 3 +- 7 files changed, 174 insertions(+), 18 deletions(-) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index a9363060..6f4dbe3c 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -2,6 +2,7 @@ # -*-coding:UTF-8 -* import os +import io import sys import uuid import redis @@ -20,6 +21,28 @@ import Correlate_object # MISP from pymisp import MISPEvent, MISPObject, PyMISP +def is_valid_obj_to_export(obj_type, obj_subtype, obj_id): + if not Correlate_object.is_valid_object_type(obj_type): + return False + if not Correlate_object.is_valid_object_subtype(obj_type, obj_subtype): + return False + if not Correlate_object.exist_object(obj_type, obj_id, type_id=obj_subtype): + return False + return True + +def sanitize_obj_export_lvl(lvl): + try: + lvl = int(lvl) + except: + lvl = 0 + return lvl + +def get_export_filename(json_content): + print(json_content) + return 'ail_export.json' + +def create_in_memory_file(json_content): + return io.BytesIO(json_content.encode()) def tag_misp_object_attributes(l_ref_obj_attr, tags): for obj_attr in l_ref_obj_attr: @@ -144,12 +167,6 @@ def filter_obj_linked(l_obj): res = Correlate_object.get_object_correlation(obj['type'], obj['id'], obj.get('subtype', None)) print(res) - -def export_object_list(l_obj, mode='union'): - # filter elements to export - if mode=='linked': - filter_obj_linked(l_obj) - def add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj): global_id = Correlate_object.get_obj_global_id(dict_obj['type'], dict_obj['id'], dict_obj.get('subtype', None)) global_id_new = Correlate_object.get_obj_global_id(dict_new_obj['type'], dict_new_obj['id'], dict_new_obj.get('subtype', None)) @@ -194,7 +211,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl) add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl) -def create_list_of_objs_to_export(l_obj, mode='union'): +def create_list_of_objs_to_export(l_obj): all_obj_to_export = set() set_relationship = set() for obj in l_obj: @@ -219,10 +236,9 @@ def create_list_of_objs_to_export(l_obj, mode='union'): # add object to event event.add_object(dict_misp_obj[obj_global_id]) - print(event.to_json()) - - misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) - misp.add_event(event, pythonify=True) + #misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) + #misp.add_event(event, pythonify=True) + return event.to_json() def create_all_misp_obj(all_obj_to_export, set_relationship): diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index b60cebae..4dbfdd20 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -24,7 +24,22 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None def is_valid_object_type(object_type): - if object_type in ['domain', 'item', 'image', 'decoded']: + if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']: + return True + else: + return False + +def is_valid_object_subtype(object_type, object_subtype): + if object_type == 'pgp': + return Pgp.pgp.is_valid_obj_subtype(object_subtype) + elif object_type == 'cryptocurrency': + return Pgp.pgp.is_valid_obj_subtype(object_subtype) + elif object_subtype == None: + return True + else: + return False + + if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']: return True else: return False diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index d7bbe941..891373a4 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -133,6 +133,12 @@ class Correlation(object): ''' return self.all_correlation_types + def is_valid_obj_subtype(self, subtype): + if subtype in self.all_correlation_types: + return True + else: + return False + def get_correlation_obj_type(self): if self.correlation_name=='pgpdump': return 'pgp' diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index f93b7cd3..1c9ff702 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -10,7 +10,7 @@ import sys import json import random -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -73,3 +73,45 @@ def import_object_file(): def export_object(): object_type = request.args.get('object_type') return render_template("export_object.html", bootstrap_label=bootstrap_label) + +@import_export.route("/import_export/export_file", methods=['POST']) +@login_required +@login_analyst +def export_object_file(): + l_obj_to_export = [] + l_obj_invalid = [] + for obj_tuple in list(request.form): + l_input = request.form.getlist(obj_tuple) + if len(l_input) == 3: + obj_type = l_input[0] + obj_id = l_input[1] + lvl = l_input[2] + lvl = MispExport.sanitize_obj_export_lvl(lvl) + + obj_subtype = obj_type.split(';') + if len(obj_subtype) == 2: + obj_type = obj_subtype[0] + obj_subtype = obj_subtype[1] + else: + obj_subtype = None + + obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl} + if obj_subtype: + obj_dict['subtype'] = obj_subtype + + if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id): + l_obj_to_export.append(obj_dict) + else: + l_obj_invalid.append(obj_dict) + print(l_obj_to_export) + print(l_obj_invalid) + + if l_obj_to_export: + + json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export) + export_filename = MispExport.get_export_filename(json_export) + json_export = MispExport.create_in_memory_file(json_export) + return send_file(json_export, as_attachment=True, attachment_filename=export_filename) + + else: + return render_template("export_object.html", bootstrap_label=bootstrap_label) diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html index a1794296..7553a2d1 100644 --- a/var/www/templates/decoded/menu_sidebar.html +++ b/var/www/templates/decoded/menu_sidebar.html @@ -40,12 +40,12 @@