mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 07:47:17 +00:00
chg: [UI MISP export] add object export picker by user
This commit is contained in:
Terrtia
parent
62e3d95f22
commit
301e7e67d1
17 changed files with 130 additions and 96 deletions
|
|
@ -66,7 +66,7 @@ def search_crytocurrency(item_id, item_content):
|
|||
is_valid_crypto_addr = True
|
||||
print('{} address found : {}'.format(crypto_name, address))
|
||||
# build bitcoin correlation
|
||||
Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
|
||||
Cryptocurrency.cryptocurrency.save_item_correlation(crypto_name, address, item_id, Item.get_item_date(item_id))
|
||||
|
||||
# At least one valid cryptocurrency address was found
|
||||
if(is_valid_crypto_addr):
|
||||
|
|
|
|||
|
|
@ -50,24 +50,30 @@ def decode_string(content, item_id, item_date, encoded_list, decoder_name, encod
|
|||
find = True
|
||||
|
||||
sha1_string = sha1(decoded_file).hexdigest()
|
||||
mimetype = Decoded.get_file_mimetype(file_content)
|
||||
mimetype = Decoded.get_file_mimetype(decoded_file)
|
||||
if not mimetype:
|
||||
print(item_id)
|
||||
print(sha1_string)
|
||||
raise Exception('Invalid mimetype')
|
||||
Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype)
|
||||
Decoded.save_item_relationship(sha1_string, item_id)
|
||||
Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name)
|
||||
|
||||
#remove encoded from item content
|
||||
content = content.replace(encoded, '', 1)
|
||||
|
||||
print('{} : {} - {}'.format(item_id, decoder_name, mimetype))
|
||||
if(find):
|
||||
set_out_item(decoder_name, message)
|
||||
set_out_item(decoder_name, item_id)
|
||||
|
||||
return content
|
||||
|
||||
def set_out_item(decoder_name, message):
|
||||
def set_out_item(decoder_name, item_id):
|
||||
publisher.warning(decoder_name+' decoded')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
p.populate_set_out(item_id, 'Duplicate')
|
||||
|
||||
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
|
||||
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(item_id)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ if __name__ == "__main__":
|
|||
|
||||
p = Process(config_section)
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
|
||||
|
||||
maximum_month_range = int(p.config.get("Modules_Duplicates", "maximum_month_range"))
|
||||
threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep"))
|
||||
threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh"))
|
||||
|
|
@ -142,6 +144,7 @@ if __name__ == "__main__":
|
|||
paste_date = paste_date
|
||||
paste_date = paste_date if paste_date != None else "No date available"
|
||||
if paste_path != None:
|
||||
paste_path = paste_path.replace(PASTES_FOLDER+'/', '', 1)
|
||||
if paste_path != PST.p_rel_path:
|
||||
hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
|
||||
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ if __name__ == "__main__":
|
|||
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
|
||||
r_serv_db.set('ail:misp', True)
|
||||
print('Connected to MISP:', misp_url)
|
||||
except e:
|
||||
except Exception as e:
|
||||
flag_misp = False
|
||||
r_serv_db.set('ail:misp', False)
|
||||
print(e)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from pubsublogger import publisher
|
|||
from bs4 import BeautifulSoup
|
||||
|
||||
from Helper import Process
|
||||
from packages import Item
|
||||
from packages import Paste
|
||||
|
||||
from packages import Pgp
|
||||
|
|
@ -214,14 +215,16 @@ if __name__ == '__main__':
|
|||
extract_all_id(message, content, regex_pgp_signature)
|
||||
extract_all_id(message, content, regex_pgp_message)
|
||||
|
||||
item_date = Item.get_item_date(message)
|
||||
|
||||
for key_id in set_key:
|
||||
print(key_id)
|
||||
Pgp.save_pgp_data('key', date, message, key_id)
|
||||
Pgp.pgp.save_item_correlation('key', key_id, message, item_date)
|
||||
|
||||
for name_id in set_name:
|
||||
print(name_id)
|
||||
Pgp.save_pgp_data('name', date, message, name_id)
|
||||
Pgp.pgp.save_item_correlation('name', key_id, message, item_date)
|
||||
|
||||
for mail_id in set_mail:
|
||||
print(mail_id)
|
||||
Pgp.save_pgp_data('mail', date, message, mail_id)
|
||||
Pgp.pgp.save_item_correlation('mail', key_id, message, item_date)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import redis
|
|||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import ConfigLoader
|
||||
import Correlate_object
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
|
||||
|
|
@ -19,3 +20,30 @@ def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
|
|||
r_serv_objects.sadd('all_object:global_id', global_id)
|
||||
r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
|
||||
r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
|
||||
|
||||
def get_user_list_of_obj_to_export(user_id, add_uuid=False):
|
||||
set_objs_to_export = []
|
||||
res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
|
||||
for global_id in res:
|
||||
dict_obj = Correlate_object.get_global_id_from_id(global_id)
|
||||
dict_obj['lvl'] = int(res[global_id])
|
||||
if add_uuid:
|
||||
obj_dict['uuid'] = str(uuid.uuid4())
|
||||
set_objs_to_export.append(dict_obj)
|
||||
return set_objs_to_export
|
||||
|
||||
def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
|
||||
## TODO: check if user exist
|
||||
# # TODO: check if valid object
|
||||
# # TODO: check lvl
|
||||
global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
|
||||
return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
|
||||
|
||||
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
|
||||
## TODO: check if user exist
|
||||
global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
|
||||
r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
|
||||
|
||||
def delete_all_user_object_to_export(user_id):
|
||||
## TODO: check if user exist
|
||||
r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ def sanitize_obj_export_lvl(lvl):
|
|||
return lvl
|
||||
|
||||
def get_export_filename(json_content):
|
||||
return 'ail_export.json'
|
||||
return 'ail_export{}.json'.format(str(uuid.uuid4()))
|
||||
|
||||
def create_in_memory_file(json_content):
|
||||
return io.BytesIO(json_content.encode())
|
||||
|
|
|
|||
|
|
@ -149,6 +149,8 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
|
|||
|
||||
elif correlation_name == 'decoded':
|
||||
node_color = '#88CCEE'
|
||||
print(Decoded.get_decoded_item_type(value))
|
||||
print(value)
|
||||
correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
|
||||
if correlation_type == 'application':
|
||||
icon_text = '\uf15b'
|
||||
|
|
|
|||
|
|
@ -377,7 +377,6 @@ class Paste(object):
|
|||
Save an attribute as a field
|
||||
"""
|
||||
for tuple in value:
|
||||
tuple = tuple.replace(PASTES_FOLDER, '', 1)
|
||||
self.store_metadata.sadd('dup:'+self.p_path, tuple)
|
||||
|
||||
def save_others_pastes_attribute_duplicate(self, list_value):
|
||||
|
|
@ -385,8 +384,8 @@ class Paste(object):
|
|||
Save a new duplicate on others pastes
|
||||
"""
|
||||
for hash_type, path, percent, date in list_value:
|
||||
path = path.replace(PASTES_FOLDER, '', 1)
|
||||
to_add = (hash_type, self.p_path, percent, date)
|
||||
path = path.replace(self.PASTES_FOLDER, '', 1)
|
||||
to_add = (hash_type, self.p_rel_path, percent, date)
|
||||
self.store_metadata.sadd('dup:'+path,to_add)
|
||||
|
||||
def _get_from_redis(self, r_serv):
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import sys
|
|||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Correlation
|
||||
import Item
|
||||
import Correlation
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -20,10 +20,10 @@ pgp = Correlation.Correlation('pgpdump', ['key', 'mail', 'name'])
|
|||
|
||||
def get_pgp(request_dict, pgp_type):
|
||||
# basic verification
|
||||
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
|
||||
res = pgp.verify_correlation_field_request(request_dict, pgp_type)
|
||||
if res:
|
||||
return res
|
||||
# cerify address
|
||||
field_name = request_dict.get(pgp_type)
|
||||
|
||||
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
|
||||
return pgp.get_correlation(request_dict, pgp_type, field_name)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import MispExport
|
|||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Correlate_object
|
||||
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
import AILObjects
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
|
||||
|
|
@ -82,18 +82,27 @@ def import_object_file():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def export_object():
|
||||
return render_template("export_object.html")
|
||||
user_id = current_user.get_id()
|
||||
l_obj_to_export = set()
|
||||
|
||||
# get user saved obj to export
|
||||
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
|
||||
|
||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
|
||||
|
||||
@import_export.route("/import_export/export_file", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def export_object_file():
|
||||
user_id = current_user.get_id()
|
||||
|
||||
l_obj_to_export = []
|
||||
l_obj_invalid = []
|
||||
|
||||
export_to_misp = False
|
||||
dict_misp_event_export = {}
|
||||
|
||||
# Get new added Object
|
||||
for obj_tuple in list(request.form):
|
||||
l_input = request.form.getlist(obj_tuple)
|
||||
if len(l_input) == 3:
|
||||
|
|
@ -115,6 +124,7 @@ def export_object_file():
|
|||
|
||||
if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
|
||||
l_obj_to_export.append(obj_dict)
|
||||
AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
|
||||
else:
|
||||
if obj_id:
|
||||
l_obj_invalid.append(obj_dict)
|
||||
|
|
@ -130,6 +140,10 @@ def export_object_file():
|
|||
for obj_dict in l_obj_to_export:
|
||||
obj_dict['uuid'] = str(uuid.uuid4())
|
||||
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
|
||||
|
||||
# get user saved obj to export # # TODO: # performance
|
||||
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
|
||||
|
||||
for obj_dict in l_obj_invalid:
|
||||
obj_dict['uuid'] = str(uuid.uuid4())
|
||||
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
|
||||
|
|
@ -144,11 +158,44 @@ def export_object_file():
|
|||
publish=dict_misp_event_export.get('misp_publish', None),
|
||||
analysis=dict_misp_event_export.get('misp_event_analysis', None),
|
||||
event_info=dict_misp_event_export.get('misp_event_info', None))
|
||||
|
||||
AILObjects.delete_all_user_object_to_export(user_id)
|
||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
||||
event_metadata=event_metadata,
|
||||
l_obj_invalid=[], dict_misp_event_export=[])
|
||||
else:
|
||||
# get user saved obj to export # # TODO: # performance
|
||||
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
|
||||
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
|
||||
export_filename = MispExport.get_export_filename(json_export)
|
||||
json_export = MispExport.create_in_memory_file(json_export)
|
||||
AILObjects.delete_all_user_object_to_export(user_id)
|
||||
return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
|
||||
|
||||
|
||||
@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_object_id_to_export():
|
||||
user_id = current_user.get_id()
|
||||
user_id = current_user.get_id()
|
||||
obj_type = request.args.get('obj_type')
|
||||
obj_id = request.args.get('obj_id')
|
||||
obj_subtype = request.args.get('obj_subtype')
|
||||
obj_lvl = request.args.get('obj_lvl')
|
||||
AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
|
||||
# redirect
|
||||
return redirect(url_for('import_export.export_object'))
|
||||
|
||||
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_analyst
|
||||
# def delete_object_id_to_export():
|
||||
# user_id = current_user.get_id()
|
||||
# obj_type = request.args.get('obj_type')
|
||||
# obj_id = request.args.get('obj_id')
|
||||
# obj_subtype = request.args.get('obj_subtype')
|
||||
# obj_lvl = request.args.get('obj_lvl')
|
||||
# AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
|
||||
# # redirect
|
||||
# return 'ok'
|
||||
|
|
|
|||
|
|
@ -910,6 +910,7 @@ def range_type_json():
|
|||
if l_hash:
|
||||
for hash in l_hash:
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
if estimated_type:
|
||||
all_type.add(estimated_type)
|
||||
|
||||
range_type = []
|
||||
|
|
|
|||
|
|
@ -199,82 +199,9 @@
|
|||
</table>
|
||||
|
||||
<div>
|
||||
{% if misp %}
|
||||
<button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
|
||||
Create
|
||||
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
|
||||
Event
|
||||
</button>
|
||||
<!-- Modal MISP-->
|
||||
<div class="modal fade" id="myModal_MISP" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<div class="modal-content">
|
||||
<div class="modal-header text-center">
|
||||
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" >
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
|
||||
<form method="post" action="{{ url_for('PasteSubmit.create_misp_event') }}" target="_blank">
|
||||
|
||||
<div class="input select required">
|
||||
<label for="EventDistribution">Distribution</label>
|
||||
<select name="misp_data[Event][distribution]" id="EventDistribution" required="required">
|
||||
<option value="0" selected="selected">Your organisation only</option>
|
||||
<option value="1">This community only</option>
|
||||
<option value="2">Connected communities</option>
|
||||
<option value="3">All communities</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="input clear required">
|
||||
<label for="EventThreatLevelId">Threat Level</label>
|
||||
<select name="misp_data[Event][threat_level_id]" id="EventThreatLevelId" required="required">
|
||||
<option value="1">High</option>
|
||||
<option value="2" selected="selected">Medium</option>
|
||||
<option value="3">Low</option>
|
||||
<option value="4">Undefined</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="input select required">
|
||||
<label for="EventAnalysis">Analysis</label>
|
||||
<select name="misp_data[Event][analysis]" id="EventAnalysis" required="required">
|
||||
<option value="0">Initial</option>
|
||||
<option value="1">Ongoing</option>
|
||||
<option value="2">Completed</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<label for="EventInfo">Event Info</label>
|
||||
<input name="misp_data[Event][info]" class="form-control span6" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo" required="required"/>
|
||||
</div>
|
||||
<div class="clear">
|
||||
<label for="EventInfo">Publish Event</label>
|
||||
<input type="checkbox" value="True" id="misp_publish" name="misp_publish" >
|
||||
</div>
|
||||
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
|
||||
|
||||
{% if misp_eventid %}
|
||||
<br>
|
||||
<div class="list-group" id="misp_event">
|
||||
<li class="list-group-item active">MISP Events already Created</li>
|
||||
<a target="_blank" href="{{ misp_url }}" class="list-group-item">{{ misp_url }}</a></div>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
<div class="modal-footer center">
|
||||
<button class="btn btn-primary">
|
||||
<span class="glyphicon glyphicon-ok "></span> Create Event
|
||||
</button>
|
||||
|
||||
</form>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%}
|
||||
{% include 'import_export/block_add_user_object_to_export.html' %}
|
||||
{% endwith %}
|
||||
|
||||
{% if hive %}
|
||||
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
|
||||
|
|
|
|||
|
|
@ -111,6 +111,11 @@
|
|||
<div class="card mb-3">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-project-diagram"></i> Graph
|
||||
<span class="float-right">
|
||||
{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
|
||||
{% include 'import_export/block_add_user_object_to_export.html' %}
|
||||
{% endwith %}
|
||||
</span>
|
||||
<span class="float-right">
|
||||
<button class="btn btn-primary py-1" onclick="resize_graph();">
|
||||
<i class="fas fa-sync"></i> Resize Graph
|
||||
|
|
|
|||
|
|
@ -114,6 +114,12 @@
|
|||
{%endif%}
|
||||
{%endif%}
|
||||
|
||||
<div class="mt-2">
|
||||
{% with obj_type='domain', obj_id=dict_domain['domain'], obj_lvl=0%}
|
||||
{% include 'import_export/block_add_user_object_to_export.html' %}
|
||||
{% endwith %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
|
||||
Add to
|
||||
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
|
||||
Export
|
||||
</a>
|
||||
|
|
@ -165,7 +165,9 @@ $('.add-field').click(function() {
|
|||
});
|
||||
|
||||
$('.fields').on('click', '.delete-field', function(){
|
||||
console.log($(this).parent());
|
||||
$(this).parent().remove();
|
||||
//$.get( "#")
|
||||
});
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue