chg: [UI MISP export] export to MISP instance

OVERVIEW.md

@@ -23,6 +23,7 @@ Redis and ARDB overview
      DB 7 - Metadata
      DB 8 - Statistics
      DB 9 - Crawler
+     DB 10 - Objects
 
 * ARDB on TCP port <year>
     - DB 0 - Lines duplicate

bin/export/AILObjects.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import uuid
+import redis
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
+config_loader = None
+
+def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
+    r_serv_objects.sadd('all_object:uuid', obj_uuid)
+    r_serv_objects.sadd('all_object:global_id', global_id)
+    r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
+    r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)

bin/export/MispExport.py

@@ -18,6 +18,10 @@ import Screenshot
 
 import Correlate_object
 
+# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
+sys.path.append('../../configs/keys')
+from mispKEYS import misp_url, misp_key, misp_verifycert
+
 # MISP
 from pymisp import MISPEvent, MISPObject, PyMISP
 
@@ -38,7 +42,6 @@ def sanitize_obj_export_lvl(lvl):
     return lvl
 
 def get_export_filename(json_content):
-    print(json_content)
     return 'ail_export.json'
 
 def create_in_memory_file(json_content):
@@ -211,7 +214,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
         add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
 
 
-def create_list_of_objs_to_export(l_obj):
+def create_list_of_objs_to_export(l_obj, r_type='json'):
     all_obj_to_export = set()
     set_relationship = set()
     for obj in l_obj:
@@ -236,10 +239,10 @@ def create_list_of_objs_to_export(l_obj):
             # add object to event
             event.add_object(dict_misp_obj[obj_global_id])
 
-    #misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
-    #misp.add_event(event, pythonify=True)
+    if r_type == 'json':
         return event.to_json()
-
+    else:
+        return event
 
 def create_all_misp_obj(all_obj_to_export, set_relationship):
     dict_misp_obj = {}
@@ -322,9 +325,65 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
             return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
     return None
 
+def sanitize_event_distribution(distribution):
+    try:
+        int(distribution)
+        if (0 <= distribution <= 3):
+            return distribution
+        else:
+            return 0
+    except:
+        return 0
+
+def sanitize_event_threat_level_id(threat_level_id):
+    try:
+        int(threat_level_id)
+        if (1 <= threat_level_id <= 4):
+            return threat_level_id
+        else:
+            return 4
+    except:
+        return 4
+
+def sanitize_event_analysis(analysis):
+    try:
+        int(analysis)
+        if (0 <= analysis <= 2):
+            return analysis
+        else:
+            return 0
+    except:
+        return 0
+
+def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
+    if event_info:
+        event.info = event_info
+    event.distribution = sanitize_event_distribution(distribution)
+    event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
+    event.analysis = sanitize_event_analysis(analysis)
+    if publish:
+        event.publish()
+
+    # # TODO: handle multiple MISP instance
+    misp = PyMISP(misp_url, misp_key, misp_verifycert)
+    misp_event = misp.add_event(event, pythonify=True)
+    # # TODO: handle error
+    event_metadata = extract_event_metadata(misp_event)
+    return event_metadata
+
+def extract_event_metadata(event):
+    event_metadata = {}
+    event_metadata['uuid'] = event.uuid
+    event_metadata['id'] = event.id
+    if misp_url[-1] == '/':
+        event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id'])
+    else:
+        event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id'])
+    return event_metadata
+
 ######
 #
-# EXPORT LVL DEFINITION:
+# EXPORT LVL DEFINITION: (== Correl<tion DEPTH)
 #
 # LVL 0 => PARTIAL    Only add core item Correlation
 # LVL 1 => DETAILED   Also add correlated_items correlation

bin/export/MispImport.py

@@ -16,7 +16,7 @@ import Domain
 import Screenshot
 import Correlate_object
 
-import Import
+import AILObjects
 
 # MISP
 from pymisp import MISPEvent, MISPObject, PyMISP
@@ -209,11 +209,7 @@ def create_obj_relationships(map_uuid_global_id, misp_obj):
 
 def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
     for obj_uuid in map_uuid_global_id:
-        create_map_objuuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
-
-def create_map_objuuid_golbal_id(obj_uuid, global_id):
-    print(obj_uuid)
-    print(global_id)
+        AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
 
 def import_objs_from_file(filepath):
     map_uuid_global_id = {}

configs/core.cfg.sample

@@ -211,6 +211,11 @@ host = localhost
 port = 6382
 db = 9
 
+[ARDB_Objects]
+host = localhost
+port = 6382
+db = 10
+
 [Url]
 cc_critical = DE
 

var/www/blueprints/import_export.py

@@ -90,6 +90,10 @@ def export_object():
 def export_object_file():
     l_obj_to_export = []
     l_obj_invalid = []
+
+    export_to_misp = False
+    dict_misp_event_export = {}
+
     for obj_tuple in list(request.form):
         l_input = request.form.getlist(obj_tuple)
         if len(l_input) == 3:
@@ -114,6 +118,13 @@ def export_object_file():
             else:
                 if obj_id:
                     l_obj_invalid.append(obj_dict)
+        else:
+            dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
+
+    if dict_misp_event_export.get('export_to_misp', None):
+        export_to_misp = True
+    else:
+        dict_misp_event_export = None
 
     if l_obj_invalid:
         for obj_dict in l_obj_to_export:
@@ -124,9 +135,19 @@ def export_object_file():
             obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
 
         return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
-                                l_obj_invalid=l_obj_invalid)
+                                l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)
+    else:
+        if export_to_misp:
+            event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event')
+            event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None),
+                                        threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None),
+                                        publish=dict_misp_event_export.get('misp_publish', None),
+                                        analysis=dict_misp_event_export.get('misp_event_analysis', None),
+                                        event_info=dict_misp_event_export.get('misp_event_info', None))
+            return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
+                                    event_metadata=event_metadata,
+                                    l_obj_invalid=[], dict_misp_event_export=[])
         else:
-
             json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
             export_filename = MispExport.get_export_filename(json_export)
             json_export = MispExport.create_in_memory_file(json_export)

var/www/templates/import_export/block_create_misp_event.html

@@ -0,0 +1,46 @@
+<div class="form-group row">
+  <label class="col-form-labe col-sm-2" for="EventDistribution">Distribution: </label>
+	<div class="col-sm-10">
+	  <select name="misp_event_distribution" id="EventDistribution">
+	    <option value="0" selected="selected">Your organisation only</option>
+	    <option value="1">This community only</option>
+	    <option value="2">Connected communities</option>
+	    <option value="3">All communities</option>
+		</select>
+	</div>
+</div>
+
+<div class="form-group row">
+	<label class="col-form-label col-sm-2" for="EventThreatLevelId">Threat Level: </label>
+	<div class="col-sm-10">
+		<select name="misp_threat_level_id" id="EventThreatLevelId">
+			<option value="1">High</option>
+			<option value="2" selected="selected">Medium</option>
+			<option value="3">Low</option>
+			<option value="4">Undefined</option>
+		</select>
+	</div>
+</div>
+
+<div class="form-group row">
+	<label class="col-form-label col-sm-2" for="EventAnalysis">Analysis: </label>
+	<div class="col-sm-10">
+		<select name="misp_event_analysis" id="EventAnalysis">
+			<option value="0">Initial</option>
+			<option value="1">Ongoing</option>
+			<option value="2">Completed</option>
+		</select>
+	</div>
+</div>
+
+<div class="form-group row">
+	<label class="col-form-label col-sm-2" for="EventInfo">Event Info: </label>
+	<div class="col-sm-10">
+		<input name="misp_event_info" class="form-control" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo">
+	</div>
+</div>
+
+<div>
+	<label class="col-form-label" for="EventInfo">Publish Event </label>
+	<input type="checkbox" value="True" id="misp_publish" name="misp_publish">
+</div>

var/www/templates/import_export/export_object.html

@@ -28,7 +28,11 @@
 
 				<div class="col-12 col-lg-10" id="core_content">
 
-					<h3>MISP exporter</h3>
+					<div class="card mb-3 mt-1">
+						<div class="card-header text-white bg-dark">
+							<h5 class="card-title"><img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:100px;"> MISP Exporter</h5>
+						</div>
+						<div class="card-body">
 
 							<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
 
@@ -88,8 +92,23 @@
 								</div>
 
 							</div>
+
+							<div class="d-flex mb-3">
+								JSON Export&nbsp;
+								<div class="custom-control custom-switch">
+									<input class="custom-control-input" type="checkbox" name="export_to_misp" value="True" id="export_to_misp">
+									<label class="custom-control-label" for="export_to_misp">
+										Export to MISP Instance
+									</label>
+								</div>
+							</div>
+
+							<div class="card border-dark my-3 px-3 py-3" id="export_to_misp_div">
+								{% include 'import_export/block_create_misp_event.html' %}
+							</div>
+
 							<div class="form-group">
-							<button class="btn btn-info" name="submit" type="submit">Export Objects</button>
+								<button class="btn btn-info" type="submit">Export Objects</button>
 							</div>
 
 							</form>
@@ -97,10 +116,25 @@
 						</div>
 					</div>
 
+					{% if event_metadata %}
+						MISP Event Created:
+						<a target="_blank" href="{{  event_metadata['url'] }}">
+							{{  event_metadata['url'] }}
+						</a>
+					{% endif %}
+
+			</div>
+		</div>
+
 <script>
 $(document).ready(function(){
 		$("#page-Decoded").addClass("active");
 		$("#nav_misp_export").addClass("active");
+		export_to_misp_input_controler();
+
+		$('#export_to_misp').change(function () {
+			export_to_misp_input_controler();
+		});
 });
 
 function toggle_sidebar(){
@@ -141,6 +175,14 @@ function uuidv4() {
   );
 }
 
+function export_to_misp_input_controler() {
+	if($('#export_to_misp').is(':checked')){
+			$("#export_to_misp_div").show();
+		}else{
+			$("#export_to_misp_div").hide();
+		}
+}
+
 </script>