From c7eeae2a44cc502f7d4d038a947d92db7ba17c53 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 6 Apr 2023 15:13:27 +0200 Subject: [PATCH] chg: [correlation] new admin enpoint delete correlation + add screenshot correlation filter --- bin/crawlers/Crawler.py | 10 ++++++---- bin/lib/correlations_engine.py | 7 +++++++ bin/lib/objects/ail_objects.py | 13 ++++++++----- var/www/blueprints/correlation.py | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index eb279674..fb309906 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -108,7 +108,7 @@ class Crawler(AbstractModule): if capture: try: status = self.lacus.get_capture_status(capture.uuid) - if status != crawlers.CaptureStatus.DONE: # TODO ADD GLOBAL TIMEOUT-> Save start time + if status != crawlers.CaptureStatus.DONE: # TODO ADD GLOBAL TIMEOUT-> Save start time ### print start time capture.update(status) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) else: @@ -248,9 +248,11 @@ class Crawler(AbstractModule): if 'png' in entries and entries['png']: screenshot = Screenshots.create_screenshot(entries['png'], b64=False) if screenshot: - # Create Correlations - screenshot.add_correlation('item', '', item_id) - screenshot.add_correlation('domain', '', self.domain.id) + # Remove Errors pages # TODO Replace with warning list ??? + if screenshot.id not in ['27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748']: + # Create Correlations + screenshot.add_correlation('item', '', item_id) + screenshot.add_correlation('domain', '', self.domain.id) # HAR if self.har: if 'har' in entries and entries['har']: diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 27307ad4..f1a2f3b7 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -118,6 +118,13 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}') r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}') +def delete_obj_correlations(obj_type, subtype, obj_id): + obj_correlations = get_correlations(obj_type, subtype, obj_id) + for correl_type in obj_correlations: + for str_obj in obj_correlations[correl_type]: + subtype2, obj2_id = str_obj.split(':', 1) + delete_obj_correlation(obj_type, subtype, obj_id, correl_type, subtype2, obj2_id) + # # bypass max result/objects ??? # def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300): # objs = set() diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 9cf87eaf..8040b721 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -28,8 +28,6 @@ config_loader = ConfigLoader() config_loader = None - - def is_valid_object_type(obj_type): return obj_type in get_all_objects() @@ -138,7 +136,7 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False): return meta -def get_objects_meta(objs, options=[], flask_context=False): +def get_objects_meta(objs, options=set(), flask_context=False): metas = [] for obj_dict in objs: metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options, @@ -166,9 +164,9 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): def get_ui_obj_tag_table_keys(obj_type): - ''' + """ Warning: use only in flask (dynamic templates) - ''' + """ if obj_type == "domain": return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot @@ -304,6 +302,11 @@ def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types ################################################################################ ################################################################################ +def delete_obj_correlations(obj_type, subtype, obj_id): + obj = get_object(obj_type, subtype, obj_id) + if obj.exists(): + return correlations_engine.delete_obj_correlations(obj_type, subtype, obj_id) + def delete_obj(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) return obj.delete() diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 7e8a11a6..96ac28b9 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -174,9 +174,23 @@ def graph_node_json(): #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) +@correlation.route('/correlation/delete', methods=['GET']) +@login_required +@login_admin +def correlation_delete(): + obj_type = request.args.get('type') + subtype = request.args.get('subtype', '') + obj_id = request.args.get('id') + + if not ail_objects.exists_obj(obj_type, subtype, obj_id): + return abort(404) + + ail_objects.delete_obj_correlations(obj_type, subtype, obj_id) + return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id)) + @correlation.route('/correlation/tags/add', methods=['POST']) @login_required -@login_read_only +@login_analyst def correlation_tags_add(): obj_id = request.form.get('tag_obj_id') subtype = request.form.get('tag_subtype', '')