AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph

Browse source
This commit is contained in:
Terrtia 2023-07-07 16:29:32 +02:00
parent 450ebdd789
commit 482fc21b5e
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 91 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bin/lib/correlations_engine.py
View file

@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id):
subtype = ''
return f'{obj_type}:{subtype}:{obj_id}'
def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False):
links = set()
nodes = set()
meta = {'complete': True, 'objs': set()}
obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
_get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='')
_get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='')
return obj_str_id, nodes, links, meta
def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''):
def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''):
obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
meta['objs'].add(obj_str_id)
nodes.add(obj_str_id)
@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
for str_obj in obj_correlations[correl_type]:
subtype2, obj2_id = str_obj.split(':', 1)
obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
# filter objects to hide
if obj2_str_id in objs_hidden:
continue
meta['objs'].add(obj2_str_id)
if obj2_str_id == previous_str_obj:
@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
if level > 0:
next_level = level - 1
_get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id)
_get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id)

17
bin/lib/objects/ail_objects.py
View file

@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_correlations()
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden):
if len(objs) < nb_max or nb_max == 0:
if lvl == 0:
objs.add((obj_type, subtype, obj_id))
@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv
for obj2_type in correlations:
for str_obj in correlations[obj2_type]:
obj2_subtype, obj2_id = str_obj.split(':', 1)
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden:
continue # filter object to hide
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden)
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
objs = set()
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden)
return objs
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
print(nb_max)
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden)
# print(objs)
for obj_tuple in objs:
obj1_type, subtype1, id1 = obj_tuple
@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
objs_hidden=set(),
flask_context=False):
obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
filter_types=filter_types,
max_nodes=max_nodes, level=level,
objs_hidden=objs_hidden,
flask_context=flask_context)
# print(meta)
meta['objs'] = list(meta['objs'])

35
var/www/blueprints/correlation.py
View file

@ -61,6 +61,13 @@ def sanitise_level(level):
level = 2
return level
def sanitise_objs_hidden(objs_hidden):
if objs_hidden:
objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects
else:
objs_hidden = set()
return objs_hidden
# ============= ROUTES ==============
@correlation.route('/correlation/show', methods=['GET', 'POST'])
@login_required
@ -130,6 +137,10 @@ def show_correlation():
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
mode = sanitise_graph_mode(request.args.get('mode'))
level = sanitise_level(request.args.get('level'))
objs_hidden = sanitise_objs_hidden(request.args.get('hidden'))
obj_to_hide = request.args.get('hide')
if obj_to_hide:
objs_hidden.add(obj_to_hide)
related_btc = bool(request.args.get('related_btc', False))
@ -139,17 +150,24 @@ def show_correlation():
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
# object exist
else:
dict_object = {"object_type": obj_type,
"correlation_id": obj_id,
else: # TODO remove old dict key
dict_object = {"type": obj_type,
"id": obj_id,
"object_type": obj_type,
"max_nodes": max_nodes, "mode": mode, "level": level,
"filter": filter_types, "filter_str": ",".join(filter_types),
"hidden": objs_hidden, "hidden_str": ",".join(objs_hidden),
"correlation_id": obj_id,
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id,
options={'tags'}, flask_context=True),
"nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id)
}
if subtype:
dict_object["subtype"] = subtype
dict_object["metadata"]['type_id'] = subtype
else:
dict_object["subtype"] = ''
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data())
@ -194,9 +212,15 @@ def graph_node_json():
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
hidden = request.args.get('hidden')
if hidden:
hidden = set(hidden.split(','))
else:
hidden = set()
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True)
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
return jsonify(json_graph)
@ -224,6 +248,7 @@ def correlation_tags_add():
nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
level = sanitise_level(request.form.get('tag_level'))
filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
hidden = sanitise_objs_hidden(request.form.get('tag_hidden'))
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
@ -252,9 +277,11 @@ def correlation_tags_add():
if tags:
ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types,
objs_hidden=hidden,
lvl=level + 1, nb_max=nb_max)
return redirect(url_for('correlation.show_correlation',
type=obj_type, subtype=subtype, id=obj_id,
level=level,
max_nodes=nb_max,
hidden=hidden, hidden_str=",".join(hidden),
filter=",".join(filter_types)))

43
var/www/templates/correlation/show_correlation.html
View file

@ -164,6 +164,11 @@
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
</span>
<span class="float-right mt-2 mx-1">
<button class="btn btn-primary py-1" onclick="reset_graph();">
<i class="fas fa-undo"></i>&nbsp;Reset Graph
</button>
</span>
<div id="incomplete_graph" class="text-danger mt-3">
<i class="fas fa-exclamation-triangle"></i>&nbsp;Graph Incomplete, Max Nodes Reached.
</div>
@ -180,6 +185,14 @@
</div>
</div>
<p>Press <b>H</b> on an object / node to hide it.</p>
{% if dict_object["hidden"] %}
<h5>Hidden objects:</h5>
{% for obj_hidden in dict_object["hidden"] %}
{{ obj_hidden }} <br>
{% endfor %}
{% endif %}
</div>
<div class="col-xl-2">
@ -343,6 +356,7 @@
<input type="hidden" id="tag_level" name="tag_level" value="{{dict_object["level"]}}">
<input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
<input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
<input type="hidden" id="tag_hidden" name="tag_hidden" value="{{dict_object["hidden_str"]}}">
{% include 'tags/block_tags_selector.html' %}
<button class="btn btn-primary mt-2">
<i class="fas fa-tag"></i> Add Tags
@ -362,7 +376,7 @@ $(document).ready(function(){
$("#incomplete_graph").hide();
$("#page-Decoded").addClass("active");
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}");
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
{% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "decoded" %}
@ -416,9 +430,18 @@ function SubmitAddTags() {
zoom.scaleTo(svg_node, 2);
}
function reset_graph() {
window.location.href = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}"
}
</script>
<script>
var correl_link = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}&max_nodes={{ dict_object["max_nodes"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}"
{% if 'hidden_str' in dict_object %}
correl_link = correl_link + "&hidden={{ dict_object["hidden_str"] }}"
{% endif %}
var width = 400,
height = 400;
@ -445,6 +468,8 @@ var simulation = d3.forceSimulation()
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var currentObject = null;
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
@ -540,6 +565,8 @@ d3.json(url)
$("#incomplete_graph").show();
}
d3.select("body").on("keypress", keypressed)
})
@ -559,6 +586,16 @@ function doubleclick (d) {
window.open(d.url, '_blank');
}
function keypressed () {
//console.log(d3.event.keyCode)
//console.log(currentObject.id)
// hide node, H or h key
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
window.location.href = correl_link + "&hide=" + currentObject.id
}
}
function click (d) {
console.log('clicked')
}
@ -582,6 +619,8 @@ d.fy = d.y;
function mouseovered(d) {
currentObject = d;
var d3_pageX = d3.event.pageX;
var d3_pageY = d3.event.pageY;
@ -666,6 +705,8 @@ if (d.popover) {
}
function mouseouted() {
currentObject = null;
div.transition()
.duration(500)
.style("opacity", 0);