diff --git a/bin/Cryptocurrencies.py b/bin/Cryptocurrencies.py index 260ff721..4b50eaee 100755 --- a/bin/Cryptocurrencies.py +++ b/bin/Cryptocurrencies.py @@ -66,7 +66,7 @@ def search_crytocurrency(item_id, item_content): is_valid_crypto_addr = True print('{} address found : {}'.format(crypto_name, address)) # build bitcoin correlation - Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address) + Cryptocurrency.cryptocurrency.save_item_correlation(crypto_name, address, item_id, Item.get_item_date(item_id)) # At least one valid cryptocurrency address was found if(is_valid_crypto_addr): diff --git a/bin/Decoder.py b/bin/Decoder.py index 84e32178..607fbd19 100755 --- a/bin/Decoder.py +++ b/bin/Decoder.py @@ -50,24 +50,30 @@ def decode_string(content, item_id, item_date, encoded_list, decoder_name, encod find = True sha1_string = sha1(decoded_file).hexdigest() - mimetype = Decoded.get_file_mimetype(file_content) + mimetype = Decoded.get_file_mimetype(decoded_file) + if not mimetype: + print(item_id) + print(sha1_string) + raise Exception('Invalid mimetype') Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype) Decoded.save_item_relationship(sha1_string, item_id) Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name) #remove encoded from item content content = content.replace(encoded, '', 1) + + print('{} : {} - {}'.format(item_id, decoder_name, mimetype)) if(find): - set_out_item(decoder_name, message) + set_out_item(decoder_name, item_id) return content -def set_out_item(decoder_name, message): +def set_out_item(decoder_name, item_id): publisher.warning(decoder_name+' decoded') #Send to duplicate - p.populate_set_out(message, 'Duplicate') + p.populate_set_out(item_id, 'Duplicate') - msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message) + msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(item_id) p.populate_set_out(msg, 'Tags') diff --git a/bin/Duplicates.py b/bin/Duplicates.py index 611368a1..2057116b 100755 --- a/bin/Duplicates.py +++ b/bin/Duplicates.py @@ -38,6 +38,8 @@ if __name__ == "__main__": p = Process(config_section) + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) + maximum_month_range = int(p.config.get("Modules_Duplicates", "maximum_month_range")) threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep")) threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh")) @@ -142,6 +144,7 @@ if __name__ == "__main__": paste_date = paste_date paste_date = paste_date if paste_date != None else "No date available" if paste_path != None: + paste_path = paste_path.replace(PASTES_FOLDER+'/', '', 1) if paste_path != PST.p_rel_path: hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 33a8841f..0949471c 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -165,7 +165,7 @@ if __name__ == "__main__": misp_wrapper = ailleakObject.ObjectWrapper(pymisp) r_serv_db.set('ail:misp', True) print('Connected to MISP:', misp_url) - except e: + except Exception as e: flag_misp = False r_serv_db.set('ail:misp', False) print(e) diff --git a/bin/PgpDump.py b/bin/PgpDump.py index bd2173d1..95ddf345 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -19,6 +19,7 @@ from pubsublogger import publisher from bs4 import BeautifulSoup from Helper import Process +from packages import Item from packages import Paste from packages import Pgp @@ -214,14 +215,16 @@ if __name__ == '__main__': extract_all_id(message, content, regex_pgp_signature) extract_all_id(message, content, regex_pgp_message) + item_date = Item.get_item_date(message) + for key_id in set_key: print(key_id) - Pgp.save_pgp_data('key', date, message, key_id) + Pgp.pgp.save_item_correlation('key', key_id, message, item_date) for name_id in set_name: print(name_id) - Pgp.save_pgp_data('name', date, message, name_id) + Pgp.pgp.save_item_correlation('name', key_id, message, item_date) for mail_id in set_mail: print(mail_id) - Pgp.save_pgp_data('mail', date, message, mail_id) + Pgp.pgp.save_item_correlation('mail', key_id, message, item_date) diff --git a/bin/export/AILObjects.py b/bin/export/AILObjects.py index 36218dbb..15453f3d 100755 --- a/bin/export/AILObjects.py +++ b/bin/export/AILObjects.py @@ -9,6 +9,7 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import ConfigLoader +import Correlate_object config_loader = ConfigLoader.ConfigLoader() r_serv_objects = config_loader.get_redis_conn("ARDB_Objects") @@ -19,3 +20,30 @@ def create_map_obj_uuid_golbal_id(obj_uuid, global_id): r_serv_objects.sadd('all_object:global_id', global_id) r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id) r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid) + +def get_user_list_of_obj_to_export(user_id, add_uuid=False): + set_objs_to_export = [] + res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id)) + for global_id in res: + dict_obj = Correlate_object.get_global_id_from_id(global_id) + dict_obj['lvl'] = int(res[global_id]) + if add_uuid: + obj_dict['uuid'] = str(uuid.uuid4()) + set_objs_to_export.append(dict_obj) + return set_objs_to_export + +def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None): + ## TODO: check if user exist + # # TODO: check if valid object + # # TODO: check lvl + global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype) + return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl) + +def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None): + ## TODO: check if user exist + global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype) + r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id) + +def delete_all_user_object_to_export(user_id): + ## TODO: check if user exist + r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id)) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index 343509f1..507283a6 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -42,7 +42,7 @@ def sanitize_obj_export_lvl(lvl): return lvl def get_export_filename(json_content): - return 'ail_export.json' + return 'ail_export{}.json'.format(str(uuid.uuid4())) def create_in_memory_file(json_content): return io.BytesIO(json_content.encode()) diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 9491a468..0be6afe6 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -149,6 +149,8 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non elif correlation_name == 'decoded': node_color = '#88CCEE' + print(Decoded.get_decoded_item_type(value)) + print(value) correlation_type = Decoded.get_decoded_item_type(value).split('/')[0] if correlation_type == 'application': icon_text = '\uf15b' diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index c82180d9..f6695eba 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -377,7 +377,6 @@ class Paste(object): Save an attribute as a field """ for tuple in value: - tuple = tuple.replace(PASTES_FOLDER, '', 1) self.store_metadata.sadd('dup:'+self.p_path, tuple) def save_others_pastes_attribute_duplicate(self, list_value): @@ -385,8 +384,8 @@ class Paste(object): Save a new duplicate on others pastes """ for hash_type, path, percent, date in list_value: - path = path.replace(PASTES_FOLDER, '', 1) - to_add = (hash_type, self.p_path, percent, date) + path = path.replace(self.PASTES_FOLDER, '', 1) + to_add = (hash_type, self.p_rel_path, percent, date) self.store_metadata.sadd('dup:'+path,to_add) def _get_from_redis(self, r_serv): diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py index 4edef07c..f6be3cbd 100755 --- a/bin/packages/Pgp.py +++ b/bin/packages/Pgp.py @@ -6,8 +6,8 @@ import sys import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Correlation import Item +import Correlation sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -20,10 +20,10 @@ pgp = Correlation.Correlation('pgpdump', ['key', 'mail', 'name']) def get_pgp(request_dict, pgp_type): # basic verification - res = pgpdump.verify_correlation_field_request(request_dict, pgp_type) + res = pgp.verify_correlation_field_request(request_dict, pgp_type) if res: return res # cerify address field_name = request_dict.get(pgp_type) - return pgpdump.get_correlation(request_dict, pgp_type, field_name) + return pgp.get_correlation(request_dict, pgp_type, field_name) diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index 332f7180..95e9ace7 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -28,7 +28,7 @@ import MispExport sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Correlate_object -bootstrap_label = Flask_config.bootstrap_label +import AILObjects # ============ BLUEPRINT ============ import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export')) @@ -82,18 +82,27 @@ def import_object_file(): @login_required @login_analyst def export_object(): - return render_template("export_object.html") + user_id = current_user.get_id() + l_obj_to_export = set() + + # get user saved obj to export + l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) + + return render_template("export_object.html", l_obj_to_export=l_obj_to_export) @import_export.route("/import_export/export_file", methods=['POST']) @login_required @login_analyst def export_object_file(): + user_id = current_user.get_id() + l_obj_to_export = [] l_obj_invalid = [] export_to_misp = False dict_misp_event_export = {} + # Get new added Object for obj_tuple in list(request.form): l_input = request.form.getlist(obj_tuple) if len(l_input) == 3: @@ -115,6 +124,7 @@ def export_object_file(): if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id): l_obj_to_export.append(obj_dict) + AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None)) else: if obj_id: l_obj_invalid.append(obj_dict) @@ -130,6 +140,10 @@ def export_object_file(): for obj_dict in l_obj_to_export: obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) + + # get user saved obj to export # # TODO: # performance + l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) + for obj_dict in l_obj_invalid: obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) @@ -144,11 +158,44 @@ def export_object_file(): publish=dict_misp_event_export.get('misp_publish', None), analysis=dict_misp_event_export.get('misp_event_analysis', None), event_info=dict_misp_event_export.get('misp_event_info', None)) + + AILObjects.delete_all_user_object_to_export(user_id) return render_template("export_object.html", l_obj_to_export=l_obj_to_export, event_metadata=event_metadata, l_obj_invalid=[], dict_misp_event_export=[]) else: + # get user saved obj to export # # TODO: # performance + l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export) export_filename = MispExport.get_export_filename(json_export) json_export = MispExport.create_in_memory_file(json_export) + AILObjects.delete_all_user_object_to_export(user_id) return send_file(json_export, as_attachment=True, attachment_filename=export_filename) + + +@import_export.route("/import_export/add_object_id_to_export", methods=['GET']) +@login_required +@login_analyst +def add_object_id_to_export(): + user_id = current_user.get_id() + user_id = current_user.get_id() + obj_type = request.args.get('obj_type') + obj_id = request.args.get('obj_id') + obj_subtype = request.args.get('obj_subtype') + obj_lvl = request.args.get('obj_lvl') + AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype) + # redirect + return redirect(url_for('import_export.export_object')) + +# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET']) +# @login_required +# @login_analyst +# def delete_object_id_to_export(): +# user_id = current_user.get_id() +# obj_type = request.args.get('obj_type') +# obj_id = request.args.get('obj_id') +# obj_subtype = request.args.get('obj_subtype') +# obj_lvl = request.args.get('obj_lvl') +# AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype) +# # redirect +# return 'ok' diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 6981f63b..d0747f06 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -910,7 +910,8 @@ def range_type_json(): if l_hash: for hash in l_hash: estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - all_type.add(estimated_type) + if estimated_type: + all_type.add(estimated_type) range_type = [] diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index b2c0a8bf..6af60e7b 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -199,82 +199,9 @@
- {% if misp %} - - - - {% endif %} + {% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%} + {% include 'import_export/block_add_user_object_to_export.html' %} + {% endwith %} {% if hive %}
diff --git a/var/www/templates/import_export/block_add_user_object_to_export.html b/var/www/templates/import_export/block_add_user_object_to_export.html new file mode 100644 index 00000000..e05d8580 --- /dev/null +++ b/var/www/templates/import_export/block_add_user_object_to_export.html @@ -0,0 +1,5 @@ + + Add to + + Export + diff --git a/var/www/templates/import_export/export_object.html b/var/www/templates/import_export/export_object.html index f819b4ef..9481671a 100644 --- a/var/www/templates/import_export/export_object.html +++ b/var/www/templates/import_export/export_object.html @@ -165,7 +165,9 @@ $('.add-field').click(function() { }); $('.fields').on('click', '.delete-field', function(){ + console.log($(this).parent()); $(this).parent().remove(); + //$.get( "#") });