chg: [UI MISP export] add object export picker by user

This commit is contained in:
Terrtia 2020-02-19 16:15:41 +01:00
parent 62e3d95f22
commit 301e7e67d1
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
17 changed files with 130 additions and 96 deletions

View file

@ -66,7 +66,7 @@ def search_crytocurrency(item_id, item_content):
is_valid_crypto_addr = True
print('{} address found : {}'.format(crypto_name, address))
# build bitcoin correlation
Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
Cryptocurrency.cryptocurrency.save_item_correlation(crypto_name, address, item_id, Item.get_item_date(item_id))
# At least one valid cryptocurrency address was found
if(is_valid_crypto_addr):

View file

@ -50,24 +50,30 @@ def decode_string(content, item_id, item_date, encoded_list, decoder_name, encod
find = True
sha1_string = sha1(decoded_file).hexdigest()
mimetype = Decoded.get_file_mimetype(file_content)
mimetype = Decoded.get_file_mimetype(decoded_file)
if not mimetype:
print(item_id)
print(sha1_string)
raise Exception('Invalid mimetype')
Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype)
Decoded.save_item_relationship(sha1_string, item_id)
Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name)
#remove encoded from item content
content = content.replace(encoded, '', 1)
print('{} : {} - {}'.format(item_id, decoder_name, mimetype))
if(find):
set_out_item(decoder_name, message)
set_out_item(decoder_name, item_id)
return content
def set_out_item(decoder_name, message):
def set_out_item(decoder_name, item_id):
publisher.warning(decoder_name+' decoded')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
p.populate_set_out(item_id, 'Duplicate')
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(item_id)
p.populate_set_out(msg, 'Tags')

View file

@ -38,6 +38,8 @@ if __name__ == "__main__":
p = Process(config_section)
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
maximum_month_range = int(p.config.get("Modules_Duplicates", "maximum_month_range"))
threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep"))
threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh"))
@ -142,6 +144,7 @@ if __name__ == "__main__":
paste_date = paste_date
paste_date = paste_date if paste_date != None else "No date available"
if paste_path != None:
paste_path = paste_path.replace(PASTES_FOLDER+'/', '', 1)
if paste_path != PST.p_rel_path:
hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)

View file

@ -165,7 +165,7 @@ if __name__ == "__main__":
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
except e:
except Exception as e:
flag_misp = False
r_serv_db.set('ail:misp', False)
print(e)

View file

@ -19,6 +19,7 @@ from pubsublogger import publisher
from bs4 import BeautifulSoup
from Helper import Process
from packages import Item
from packages import Paste
from packages import Pgp
@ -214,14 +215,16 @@ if __name__ == '__main__':
extract_all_id(message, content, regex_pgp_signature)
extract_all_id(message, content, regex_pgp_message)
item_date = Item.get_item_date(message)
for key_id in set_key:
print(key_id)
Pgp.save_pgp_data('key', date, message, key_id)
Pgp.pgp.save_item_correlation('key', key_id, message, item_date)
for name_id in set_name:
print(name_id)
Pgp.save_pgp_data('name', date, message, name_id)
Pgp.pgp.save_item_correlation('name', key_id, message, item_date)
for mail_id in set_mail:
print(mail_id)
Pgp.save_pgp_data('mail', date, message, mail_id)
Pgp.pgp.save_item_correlation('mail', key_id, message, item_date)

View file

@ -9,6 +9,7 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import ConfigLoader
import Correlate_object
config_loader = ConfigLoader.ConfigLoader()
r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
@ -19,3 +20,30 @@ def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
r_serv_objects.sadd('all_object:global_id', global_id)
r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
def get_user_list_of_obj_to_export(user_id, add_uuid=False):
set_objs_to_export = []
res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
for global_id in res:
dict_obj = Correlate_object.get_global_id_from_id(global_id)
dict_obj['lvl'] = int(res[global_id])
if add_uuid:
obj_dict['uuid'] = str(uuid.uuid4())
set_objs_to_export.append(dict_obj)
return set_objs_to_export
def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
## TODO: check if user exist
# # TODO: check if valid object
# # TODO: check lvl
global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
## TODO: check if user exist
global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
def delete_all_user_object_to_export(user_id):
## TODO: check if user exist
r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))

View file

@ -42,7 +42,7 @@ def sanitize_obj_export_lvl(lvl):
return lvl
def get_export_filename(json_content):
return 'ail_export.json'
return 'ail_export{}.json'.format(str(uuid.uuid4()))
def create_in_memory_file(json_content):
return io.BytesIO(json_content.encode())

View file

@ -149,6 +149,8 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
elif correlation_name == 'decoded':
node_color = '#88CCEE'
print(Decoded.get_decoded_item_type(value))
print(value)
correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
if correlation_type == 'application':
icon_text = '\uf15b'

View file

@ -377,7 +377,6 @@ class Paste(object):
Save an attribute as a field
"""
for tuple in value:
tuple = tuple.replace(PASTES_FOLDER, '', 1)
self.store_metadata.sadd('dup:'+self.p_path, tuple)
def save_others_pastes_attribute_duplicate(self, list_value):
@ -385,8 +384,8 @@ class Paste(object):
Save a new duplicate on others pastes
"""
for hash_type, path, percent, date in list_value:
path = path.replace(PASTES_FOLDER, '', 1)
to_add = (hash_type, self.p_path, percent, date)
path = path.replace(self.PASTES_FOLDER, '', 1)
to_add = (hash_type, self.p_rel_path, percent, date)
self.store_metadata.sadd('dup:'+path,to_add)
def _get_from_redis(self, r_serv):

View file

@ -6,8 +6,8 @@ import sys
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Correlation
import Item
import Correlation
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -20,10 +20,10 @@ pgp = Correlation.Correlation('pgpdump', ['key', 'mail', 'name'])
def get_pgp(request_dict, pgp_type):
# basic verification
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
res = pgp.verify_correlation_field_request(request_dict, pgp_type)
if res:
return res
# cerify address
field_name = request_dict.get(pgp_type)
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
return pgp.get_correlation(request_dict, pgp_type, field_name)

View file

@ -28,7 +28,7 @@ import MispExport
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Correlate_object
bootstrap_label = Flask_config.bootstrap_label
import AILObjects
# ============ BLUEPRINT ============
import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
@ -82,18 +82,27 @@ def import_object_file():
@login_required
@login_analyst
def export_object():
return render_template("export_object.html")
user_id = current_user.get_id()
l_obj_to_export = set()
# get user saved obj to export
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
@import_export.route("/import_export/export_file", methods=['POST'])
@login_required
@login_analyst
def export_object_file():
user_id = current_user.get_id()
l_obj_to_export = []
l_obj_invalid = []
export_to_misp = False
dict_misp_event_export = {}
# Get new added Object
for obj_tuple in list(request.form):
l_input = request.form.getlist(obj_tuple)
if len(l_input) == 3:
@ -115,6 +124,7 @@ def export_object_file():
if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
l_obj_to_export.append(obj_dict)
AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
else:
if obj_id:
l_obj_invalid.append(obj_dict)
@ -130,6 +140,10 @@ def export_object_file():
for obj_dict in l_obj_to_export:
obj_dict['uuid'] = str(uuid.uuid4())
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
# get user saved obj to export # # TODO: # performance
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
for obj_dict in l_obj_invalid:
obj_dict['uuid'] = str(uuid.uuid4())
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
@ -144,11 +158,44 @@ def export_object_file():
publish=dict_misp_event_export.get('misp_publish', None),
analysis=dict_misp_event_export.get('misp_event_analysis', None),
event_info=dict_misp_event_export.get('misp_event_info', None))
AILObjects.delete_all_user_object_to_export(user_id)
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
event_metadata=event_metadata,
l_obj_invalid=[], dict_misp_event_export=[])
else:
# get user saved obj to export # # TODO: # performance
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
export_filename = MispExport.get_export_filename(json_export)
json_export = MispExport.create_in_memory_file(json_export)
AILObjects.delete_all_user_object_to_export(user_id)
return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
@login_required
@login_analyst
def add_object_id_to_export():
user_id = current_user.get_id()
user_id = current_user.get_id()
obj_type = request.args.get('obj_type')
obj_id = request.args.get('obj_id')
obj_subtype = request.args.get('obj_subtype')
obj_lvl = request.args.get('obj_lvl')
AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
# redirect
return redirect(url_for('import_export.export_object'))
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
# @login_required
# @login_analyst
# def delete_object_id_to_export():
# user_id = current_user.get_id()
# obj_type = request.args.get('obj_type')
# obj_id = request.args.get('obj_id')
# obj_subtype = request.args.get('obj_subtype')
# obj_lvl = request.args.get('obj_lvl')
# AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
# # redirect
# return 'ok'

View file

@ -910,7 +910,8 @@ def range_type_json():
if l_hash:
for hash in l_hash:
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
all_type.add(estimated_type)
if estimated_type:
all_type.add(estimated_type)
range_type = []

View file

@ -199,82 +199,9 @@
</table>
<div>
{% if misp %}
<button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
Create
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
Event
</button>
<!-- Modal MISP-->
<div class="modal fade" id="myModal_MISP" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" >
</div>
<div class="modal-body">
<form method="post" action="{{ url_for('PasteSubmit.create_misp_event') }}" target="_blank">
<div class="input select required">
<label for="EventDistribution">Distribution</label>
<select name="misp_data[Event][distribution]" id="EventDistribution" required="required">
<option value="0" selected="selected">Your organisation only</option>
<option value="1">This community only</option>
<option value="2">Connected communities</option>
<option value="3">All communities</option>
</select>
</div>
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="misp_data[Event][threat_level_id]" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
<option value="4">Undefined</option>
</select>
</div>
<div class="input select required">
<label for="EventAnalysis">Analysis</label>
<select name="misp_data[Event][analysis]" id="EventAnalysis" required="required">
<option value="0">Initial</option>
<option value="1">Ongoing</option>
<option value="2">Completed</option>
</select>
</div>
<div class="clear required">
<label for="EventInfo">Event Info</label>
<input name="misp_data[Event][info]" class="form-control span6" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo" required="required"/>
</div>
<div class="clear">
<label for="EventInfo">Publish Event</label>
<input type="checkbox" value="True" id="misp_publish" name="misp_publish" >
</div>
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
{% if misp_eventid %}
<br>
<div class="list-group" id="misp_event">
<li class="list-group-item active">MISP Events already Created</li>
<a target="_blank" href="{{ misp_url }}" class="list-group-item">{{ misp_url }}</a></div>
{% endif %}
</div>
<div class="modal-footer center">
<button class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Create Event
</button>
</form>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% endif %}
{% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
{% if hive %}
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">

View file

@ -111,6 +111,11 @@
<div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
<span class="float-right">
{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
</span>
<span class="float-right">
<button class="btn btn-primary py-1" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph

View file

@ -114,6 +114,12 @@
{%endif%}
{%endif%}
<div class="mt-2">
{% with obj_type='domain', obj_id=dict_domain['domain'], obj_lvl=0%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
</div>
</div>
</div>

View file

@ -0,0 +1,5 @@
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
Add to
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
Export
</a>

View file

@ -165,7 +165,9 @@ $('.add-field').click(function() {
});
$('.fields').on('click', '.delete-field', function(){
console.log($(this).parent());
$(this).parent().remove();
//$.get( "#")
});