mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [UI MISP export] export to MISP instance
This commit is contained in:
parent
b0dde78c2e
commit
b8342cfa69
8 changed files with 264 additions and 73 deletions
|
@ -23,6 +23,7 @@ Redis and ARDB overview
|
||||||
DB 7 - Metadata
|
DB 7 - Metadata
|
||||||
DB 8 - Statistics
|
DB 8 - Statistics
|
||||||
DB 9 - Crawler
|
DB 9 - Crawler
|
||||||
|
DB 10 - Objects
|
||||||
|
|
||||||
* ARDB on TCP port <year>
|
* ARDB on TCP port <year>
|
||||||
- DB 0 - Lines duplicate
|
- DB 0 - Lines duplicate
|
||||||
|
|
21
bin/export/AILObjects.py
Executable file
21
bin/export/AILObjects.py
Executable file
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
import redis
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
|
||||||
|
r_serv_objects.sadd('all_object:uuid', obj_uuid)
|
||||||
|
r_serv_objects.sadd('all_object:global_id', global_id)
|
||||||
|
r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
|
||||||
|
r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
|
|
@ -18,6 +18,10 @@ import Screenshot
|
||||||
|
|
||||||
import Correlate_object
|
import Correlate_object
|
||||||
|
|
||||||
|
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
|
||||||
|
sys.path.append('../../configs/keys')
|
||||||
|
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||||
|
|
||||||
# MISP
|
# MISP
|
||||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||||
|
|
||||||
|
@ -38,7 +42,6 @@ def sanitize_obj_export_lvl(lvl):
|
||||||
return lvl
|
return lvl
|
||||||
|
|
||||||
def get_export_filename(json_content):
|
def get_export_filename(json_content):
|
||||||
print(json_content)
|
|
||||||
return 'ail_export.json'
|
return 'ail_export.json'
|
||||||
|
|
||||||
def create_in_memory_file(json_content):
|
def create_in_memory_file(json_content):
|
||||||
|
@ -211,7 +214,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
|
||||||
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
|
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
|
||||||
|
|
||||||
|
|
||||||
def create_list_of_objs_to_export(l_obj):
|
def create_list_of_objs_to_export(l_obj, r_type='json'):
|
||||||
all_obj_to_export = set()
|
all_obj_to_export = set()
|
||||||
set_relationship = set()
|
set_relationship = set()
|
||||||
for obj in l_obj:
|
for obj in l_obj:
|
||||||
|
@ -236,10 +239,10 @@ def create_list_of_objs_to_export(l_obj):
|
||||||
# add object to event
|
# add object to event
|
||||||
event.add_object(dict_misp_obj[obj_global_id])
|
event.add_object(dict_misp_obj[obj_global_id])
|
||||||
|
|
||||||
#misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
|
if r_type == 'json':
|
||||||
#misp.add_event(event, pythonify=True)
|
|
||||||
return event.to_json()
|
return event.to_json()
|
||||||
|
else:
|
||||||
|
return event
|
||||||
|
|
||||||
def create_all_misp_obj(all_obj_to_export, set_relationship):
|
def create_all_misp_obj(all_obj_to_export, set_relationship):
|
||||||
dict_misp_obj = {}
|
dict_misp_obj = {}
|
||||||
|
@ -322,9 +325,65 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
|
||||||
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
|
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def sanitize_event_distribution(distribution):
|
||||||
|
try:
|
||||||
|
int(distribution)
|
||||||
|
if (0 <= distribution <= 3):
|
||||||
|
return distribution
|
||||||
|
else:
|
||||||
|
return 0
|
||||||
|
except:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def sanitize_event_threat_level_id(threat_level_id):
|
||||||
|
try:
|
||||||
|
int(threat_level_id)
|
||||||
|
if (1 <= threat_level_id <= 4):
|
||||||
|
return threat_level_id
|
||||||
|
else:
|
||||||
|
return 4
|
||||||
|
except:
|
||||||
|
return 4
|
||||||
|
|
||||||
|
def sanitize_event_analysis(analysis):
|
||||||
|
try:
|
||||||
|
int(analysis)
|
||||||
|
if (0 <= analysis <= 2):
|
||||||
|
return analysis
|
||||||
|
else:
|
||||||
|
return 0
|
||||||
|
except:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
|
||||||
|
if event_info:
|
||||||
|
event.info = event_info
|
||||||
|
event.distribution = sanitize_event_distribution(distribution)
|
||||||
|
event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
|
||||||
|
event.analysis = sanitize_event_analysis(analysis)
|
||||||
|
if publish:
|
||||||
|
event.publish()
|
||||||
|
|
||||||
|
# # TODO: handle multiple MISP instance
|
||||||
|
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||||
|
misp_event = misp.add_event(event, pythonify=True)
|
||||||
|
# # TODO: handle error
|
||||||
|
event_metadata = extract_event_metadata(misp_event)
|
||||||
|
return event_metadata
|
||||||
|
|
||||||
|
def extract_event_metadata(event):
|
||||||
|
event_metadata = {}
|
||||||
|
event_metadata['uuid'] = event.uuid
|
||||||
|
event_metadata['id'] = event.id
|
||||||
|
if misp_url[-1] == '/':
|
||||||
|
event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id'])
|
||||||
|
else:
|
||||||
|
event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id'])
|
||||||
|
return event_metadata
|
||||||
|
|
||||||
######
|
######
|
||||||
#
|
#
|
||||||
# EXPORT LVL DEFINITION:
|
# EXPORT LVL DEFINITION: (== Correl<tion DEPTH)
|
||||||
#
|
#
|
||||||
# LVL 0 => PARTIAL Only add core item Correlation
|
# LVL 0 => PARTIAL Only add core item Correlation
|
||||||
# LVL 1 => DETAILED Also add correlated_items correlation
|
# LVL 1 => DETAILED Also add correlated_items correlation
|
||||||
|
|
|
@ -16,7 +16,7 @@ import Domain
|
||||||
import Screenshot
|
import Screenshot
|
||||||
import Correlate_object
|
import Correlate_object
|
||||||
|
|
||||||
import Import
|
import AILObjects
|
||||||
|
|
||||||
# MISP
|
# MISP
|
||||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||||
|
@ -209,11 +209,7 @@ def create_obj_relationships(map_uuid_global_id, misp_obj):
|
||||||
|
|
||||||
def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
|
def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
|
||||||
for obj_uuid in map_uuid_global_id:
|
for obj_uuid in map_uuid_global_id:
|
||||||
create_map_objuuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
|
AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
|
||||||
|
|
||||||
def create_map_objuuid_golbal_id(obj_uuid, global_id):
|
|
||||||
print(obj_uuid)
|
|
||||||
print(global_id)
|
|
||||||
|
|
||||||
def import_objs_from_file(filepath):
|
def import_objs_from_file(filepath):
|
||||||
map_uuid_global_id = {}
|
map_uuid_global_id = {}
|
||||||
|
|
|
@ -211,6 +211,11 @@ host = localhost
|
||||||
port = 6382
|
port = 6382
|
||||||
db = 9
|
db = 9
|
||||||
|
|
||||||
|
[ARDB_Objects]
|
||||||
|
host = localhost
|
||||||
|
port = 6382
|
||||||
|
db = 10
|
||||||
|
|
||||||
[Url]
|
[Url]
|
||||||
cc_critical = DE
|
cc_critical = DE
|
||||||
|
|
||||||
|
|
|
@ -90,6 +90,10 @@ def export_object():
|
||||||
def export_object_file():
|
def export_object_file():
|
||||||
l_obj_to_export = []
|
l_obj_to_export = []
|
||||||
l_obj_invalid = []
|
l_obj_invalid = []
|
||||||
|
|
||||||
|
export_to_misp = False
|
||||||
|
dict_misp_event_export = {}
|
||||||
|
|
||||||
for obj_tuple in list(request.form):
|
for obj_tuple in list(request.form):
|
||||||
l_input = request.form.getlist(obj_tuple)
|
l_input = request.form.getlist(obj_tuple)
|
||||||
if len(l_input) == 3:
|
if len(l_input) == 3:
|
||||||
|
@ -114,6 +118,13 @@ def export_object_file():
|
||||||
else:
|
else:
|
||||||
if obj_id:
|
if obj_id:
|
||||||
l_obj_invalid.append(obj_dict)
|
l_obj_invalid.append(obj_dict)
|
||||||
|
else:
|
||||||
|
dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
|
||||||
|
|
||||||
|
if dict_misp_event_export.get('export_to_misp', None):
|
||||||
|
export_to_misp = True
|
||||||
|
else:
|
||||||
|
dict_misp_event_export = None
|
||||||
|
|
||||||
if l_obj_invalid:
|
if l_obj_invalid:
|
||||||
for obj_dict in l_obj_to_export:
|
for obj_dict in l_obj_to_export:
|
||||||
|
@ -124,9 +135,19 @@ def export_object_file():
|
||||||
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
|
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
|
||||||
|
|
||||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
||||||
l_obj_invalid=l_obj_invalid)
|
l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)
|
||||||
|
else:
|
||||||
|
if export_to_misp:
|
||||||
|
event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event')
|
||||||
|
event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None),
|
||||||
|
threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None),
|
||||||
|
publish=dict_misp_event_export.get('misp_publish', None),
|
||||||
|
analysis=dict_misp_event_export.get('misp_event_analysis', None),
|
||||||
|
event_info=dict_misp_event_export.get('misp_event_info', None))
|
||||||
|
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
||||||
|
event_metadata=event_metadata,
|
||||||
|
l_obj_invalid=[], dict_misp_event_export=[])
|
||||||
else:
|
else:
|
||||||
|
|
||||||
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
|
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
|
||||||
export_filename = MispExport.get_export_filename(json_export)
|
export_filename = MispExport.get_export_filename(json_export)
|
||||||
json_export = MispExport.create_in_memory_file(json_export)
|
json_export = MispExport.create_in_memory_file(json_export)
|
||||||
|
|
46
var/www/templates/import_export/block_create_misp_event.html
Normal file
46
var/www/templates/import_export/block_create_misp_event.html
Normal file
|
@ -0,0 +1,46 @@
|
||||||
|
<div class="form-group row">
|
||||||
|
<label class="col-form-labe col-sm-2" for="EventDistribution">Distribution: </label>
|
||||||
|
<div class="col-sm-10">
|
||||||
|
<select name="misp_event_distribution" id="EventDistribution">
|
||||||
|
<option value="0" selected="selected">Your organisation only</option>
|
||||||
|
<option value="1">This community only</option>
|
||||||
|
<option value="2">Connected communities</option>
|
||||||
|
<option value="3">All communities</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group row">
|
||||||
|
<label class="col-form-label col-sm-2" for="EventThreatLevelId">Threat Level: </label>
|
||||||
|
<div class="col-sm-10">
|
||||||
|
<select name="misp_threat_level_id" id="EventThreatLevelId">
|
||||||
|
<option value="1">High</option>
|
||||||
|
<option value="2" selected="selected">Medium</option>
|
||||||
|
<option value="3">Low</option>
|
||||||
|
<option value="4">Undefined</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group row">
|
||||||
|
<label class="col-form-label col-sm-2" for="EventAnalysis">Analysis: </label>
|
||||||
|
<div class="col-sm-10">
|
||||||
|
<select name="misp_event_analysis" id="EventAnalysis">
|
||||||
|
<option value="0">Initial</option>
|
||||||
|
<option value="1">Ongoing</option>
|
||||||
|
<option value="2">Completed</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group row">
|
||||||
|
<label class="col-form-label col-sm-2" for="EventInfo">Event Info: </label>
|
||||||
|
<div class="col-sm-10">
|
||||||
|
<input name="misp_event_info" class="form-control" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label class="col-form-label" for="EventInfo">Publish Event </label>
|
||||||
|
<input type="checkbox" value="True" id="misp_publish" name="misp_publish">
|
||||||
|
</div>
|
|
@ -28,7 +28,11 @@
|
||||||
|
|
||||||
<div class="col-12 col-lg-10" id="core_content">
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
<h3>MISP exporter</h3>
|
<div class="card mb-3 mt-1">
|
||||||
|
<div class="card-header text-white bg-dark">
|
||||||
|
<h5 class="card-title"><img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:100px;"> MISP Exporter</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
|
||||||
<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
|
<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
|
||||||
|
|
||||||
|
@ -88,8 +92,23 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="d-flex mb-3">
|
||||||
|
JSON Export
|
||||||
|
<div class="custom-control custom-switch">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="export_to_misp" value="True" id="export_to_misp">
|
||||||
|
<label class="custom-control-label" for="export_to_misp">
|
||||||
|
Export to MISP Instance
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card border-dark my-3 px-3 py-3" id="export_to_misp_div">
|
||||||
|
{% include 'import_export/block_create_misp_event.html' %}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<button class="btn btn-info" name="submit" type="submit">Export Objects</button>
|
<button class="btn btn-info" type="submit">Export Objects</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</form>
|
</form>
|
||||||
|
@ -97,10 +116,25 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{% if event_metadata %}
|
||||||
|
MISP Event Created:
|
||||||
|
<a target="_blank" href="{{ event_metadata['url'] }}">
|
||||||
|
{{ event_metadata['url'] }}
|
||||||
|
</a>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
$("#page-Decoded").addClass("active");
|
$("#page-Decoded").addClass("active");
|
||||||
$("#nav_misp_export").addClass("active");
|
$("#nav_misp_export").addClass("active");
|
||||||
|
export_to_misp_input_controler();
|
||||||
|
|
||||||
|
$('#export_to_misp').change(function () {
|
||||||
|
export_to_misp_input_controler();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
function toggle_sidebar(){
|
function toggle_sidebar(){
|
||||||
|
@ -141,6 +175,14 @@ function uuidv4() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function export_to_misp_input_controler() {
|
||||||
|
if($('#export_to_misp').is(':checked')){
|
||||||
|
$("#export_to_misp_div").show();
|
||||||
|
}else{
|
||||||
|
$("#export_to_misp_div").hide();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue