From 482fc21b5e93f37982c2c580fb4e710eeac4d9ff Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 7 Jul 2023 16:29:32 +0200 Subject: [PATCH] chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph --- bin/lib/correlations_engine.py | 12 ++++-- bin/lib/objects/ail_objects.py | 17 +++++--- var/www/blueprints/correlation.py | 35 +++++++++++++-- .../correlation/show_correlation.html | 43 ++++++++++++++++++- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 1a2081ac..39ccaa4e 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id): subtype = '' return f'{obj_type}:{subtype}:{obj_id}' -def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False): +def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False): links = set() nodes = set() meta = {'complete': True, 'objs': set()} obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) - _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='') + _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='') return obj_str_id, nodes, links, meta -def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''): +def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''): obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) meta['objs'].add(obj_str_id) nodes.add(obj_str_id) @@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, for str_obj in obj_correlations[correl_type]: subtype2, obj2_id = str_obj.split(':', 1) obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id) + # filter objects to hide + if obj2_str_id in objs_hidden: + continue + meta['objs'].add(obj2_str_id) if obj2_str_id == previous_str_obj: @@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, if level > 0: next_level = level - 1 - _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id) + _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 4279c776..e0eeda63 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) return obj.get_correlations() -def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): +def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden): if len(objs) < nb_max or nb_max == 0: if lvl == 0: objs.add((obj_type, subtype, obj_id)) @@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv for obj2_type in correlations: for str_obj in correlations[obj2_type]: obj2_subtype, obj2_id = str_obj.split(':', 1) - _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max) + if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden: + continue # filter object to hide + _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden) -def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300): +def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): objs = set() - _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max) + _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden) return objs -def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): - print(nb_max) - objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) +def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): + objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden) # print(objs) for obj_tuple in objs: obj1_type, subtype1, id1 = obj_tuple @@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, + objs_hidden=set(), flask_context=False): obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden, flask_context=flask_context) # print(meta) meta['objs'] = list(meta['objs']) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index d5bb1c82..594dc660 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -61,6 +61,13 @@ def sanitise_level(level): level = 2 return level +def sanitise_objs_hidden(objs_hidden): + if objs_hidden: + objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects + else: + objs_hidden = set() + return objs_hidden + # ============= ROUTES ============== @correlation.route('/correlation/show', methods=['GET', 'POST']) @login_required @@ -130,6 +137,10 @@ def show_correlation(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) mode = sanitise_graph_mode(request.args.get('mode')) level = sanitise_level(request.args.get('level')) + objs_hidden = sanitise_objs_hidden(request.args.get('hidden')) + obj_to_hide = request.args.get('hide') + if obj_to_hide: + objs_hidden.add(obj_to_hide) related_btc = bool(request.args.get('related_btc', False)) @@ -139,17 +150,24 @@ def show_correlation(): if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) # object exist - else: - dict_object = {"object_type": obj_type, - "correlation_id": obj_id, + else: # TODO remove old dict key + dict_object = {"type": obj_type, + "id": obj_id, + "object_type": obj_type, "max_nodes": max_nodes, "mode": mode, "level": level, "filter": filter_types, "filter_str": ",".join(filter_types), + "hidden": objs_hidden, "hidden_str": ",".join(objs_hidden), + + "correlation_id": obj_id, "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags'}, flask_context=True), "nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id) } if subtype: + dict_object["subtype"] = subtype dict_object["metadata"]['type_id'] = subtype + else: + dict_object["subtype"] = '' dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, tags_selector_data=Tag.get_tags_selector_data()) @@ -194,9 +212,15 @@ def graph_node_json(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) + hidden = request.args.get('hidden') + if hidden: + hidden = set(hidden.split(',')) + else: + hidden = set() + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) + json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) @@ -224,6 +248,7 @@ def correlation_tags_add(): nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max')) level = sanitise_level(request.form.get('tag_level')) filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(',')) + hidden = sanitise_objs_hidden(request.form.get('tag_hidden')) if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) @@ -252,9 +277,11 @@ def correlation_tags_add(): if tags: ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, + objs_hidden=hidden, lvl=level + 1, nb_max=nb_max) return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id, level=level, max_nodes=nb_max, + hidden=hidden, hidden_str=",".join(hidden), filter=",".join(filter_types))) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 326b637f..b243bd35 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -164,6 +164,11 @@  Resize Graph + + +
 Graph Incomplete, Max Nodes Reached.
@@ -180,6 +185,14 @@ +

Press H on an object / node to hide it.

+ {% if dict_object["hidden"] %} +
Hidden objects:
+ {% for obj_hidden in dict_object["hidden"] %} + {{ obj_hidden }}
+ {% endfor %} + {% endif %} +
@@ -343,6 +356,7 @@ + {% include 'tags/block_tags_selector.html' %}