chg: [correlation] new admin enpoint delete correlation + add screenshot correlation filter

bin/crawlers/Crawler.py

@@ -108,7 +108,7 @@ class Crawler(AbstractModule):
         if capture:
             try:
                 status = self.lacus.get_capture_status(capture.uuid)
-                if status != crawlers.CaptureStatus.DONE:  # TODO ADD GLOBAL TIMEOUT-> Save start time
+                if status != crawlers.CaptureStatus.DONE:  # TODO ADD GLOBAL TIMEOUT-> Save start time ### print start time
                     capture.update(status)
                     print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
                 else:
@@ -248,9 +248,11 @@ class Crawler(AbstractModule):
                 if 'png' in entries and entries['png']:
                     screenshot = Screenshots.create_screenshot(entries['png'], b64=False)
                     if screenshot:
-                        # Create Correlations
-                        screenshot.add_correlation('item', '', item_id)
-                        screenshot.add_correlation('domain', '', self.domain.id)
+                        # Remove Errors pages # TODO Replace with warning list ???
+                        if screenshot.id not in ['27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748']:
+                            # Create Correlations
+                            screenshot.add_correlation('item', '', item_id)
+                            screenshot.add_correlation('domain', '', self.domain.id)
             # HAR
             if self.har:
                 if 'har' in entries and entries['har']:

bin/lib/correlations_engine.py

@@ -118,6 +118,13 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob
     r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}')
     r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}')
 
+def delete_obj_correlations(obj_type, subtype, obj_id):
+    obj_correlations = get_correlations(obj_type, subtype, obj_id)
+    for correl_type in obj_correlations:
+        for str_obj in obj_correlations[correl_type]:
+            subtype2, obj2_id = str_obj.split(':', 1)
+            delete_obj_correlation(obj_type, subtype, obj_id, correl_type, subtype2, obj2_id)
+
 # # bypass max result/objects ???
 # def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300):
 #     objs = set()

bin/lib/objects/ail_objects.py

@@ -28,8 +28,6 @@ config_loader = ConfigLoader()
 config_loader = None
 
 
-
-
 def is_valid_object_type(obj_type):
     return obj_type in get_all_objects()
 
@@ -138,7 +136,7 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
     return meta
 
 
-def get_objects_meta(objs, options=[], flask_context=False):
+def get_objects_meta(objs, options=set(), flask_context=False):
     metas = []
     for obj_dict in objs:
         metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options,
@@ -166,9 +164,9 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
 
 
 def get_ui_obj_tag_table_keys(obj_type):
-    '''
+    """
     Warning: use only in flask (dynamic templates)
-    '''
+    """
     if obj_type == "domain":
         return ['id', 'first_seen', 'last_check', 'status']  # # TODO: add root screenshot
 
@@ -304,6 +302,11 @@ def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types
 ################################################################################
 ################################################################################
 
+def delete_obj_correlations(obj_type, subtype, obj_id):
+    obj = get_object(obj_type, subtype, obj_id)
+    if obj.exists():
+        return correlations_engine.delete_obj_correlations(obj_type, subtype, obj_id)
+
 def delete_obj(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.delete()

var/www/blueprints/correlation.py

@@ -174,9 +174,23 @@ def graph_node_json():
     #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
     return jsonify(json_graph)
 
+@correlation.route('/correlation/delete', methods=['GET'])
+@login_required
+@login_admin
+def correlation_delete():
+    obj_type = request.args.get('type')
+    subtype = request.args.get('subtype', '')
+    obj_id = request.args.get('id')
+
+    if not ail_objects.exists_obj(obj_type, subtype, obj_id):
+        return abort(404)
+
+    ail_objects.delete_obj_correlations(obj_type, subtype, obj_id)
+    return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id))
+
 @correlation.route('/correlation/tags/add', methods=['POST'])
 @login_required
-@login_read_only
+@login_analyst
 def correlation_tags_add():
     obj_id = request.form.get('tag_obj_id')
     subtype = request.form.get('tag_subtype', '')