ail-framework/var/www/blueprints/correlation.py

372 lines
15 KiB
Python
Raw Permalink Normal View History

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
import json
2019-11-19 08:02:25 +00:00
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort
from flask_login import login_required
sys.path.append('modules')
import Flask_config
# Import Role_Manager
2024-09-05 14:40:24 +00:00
from Role_Manager import login_admin, login_read_only
2022-08-19 14:53:31 +00:00
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects import ail_objects
from lib import chats_viewer
2023-04-05 14:09:06 +00:00
from lib import Tag
2022-08-19 14:53:31 +00:00
bootstrap_label = Flask_config.bootstrap_label
vt_enabled = Flask_config.vt_enabled
# ============ BLUEPRINT ============
2023-04-05 14:09:06 +00:00
correlation = Blueprint('correlation', __name__,
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
# ============ VARIABLES ============
# ============ FUNCTIONS ============
def sanitise_graph_mode(graph_mode):
if graph_mode not in ('inter', 'union'):
return 'union'
else:
return graph_mode
def sanitise_nb_max_nodes(nb_max_nodes):
try:
nb_max_nodes = int(nb_max_nodes)
if nb_max_nodes < 2 and nb_max_nodes != 0:
nb_max_nodes = 300
2023-04-05 14:09:06 +00:00
except (TypeError, ValueError):
nb_max_nodes = 300
return nb_max_nodes
def sanitise_level(level):
try:
level = int(level)
if level < 0:
level = 2
except (TypeError, ValueError):
level = 2
return level
def sanitise_objs_hidden(objs_hidden):
if objs_hidden:
objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects
else:
objs_hidden = set()
return objs_hidden
# ============= ROUTES ==============
2023-04-05 14:09:06 +00:00
@correlation.route('/correlation/show', methods=['GET', 'POST'])
@login_required
@login_read_only
def show_correlation():
if request.method == 'POST':
object_type = request.form.get('obj_type')
subtype = request.form.get('subtype')
obj_id = request.form.get('obj_id')
max_nodes = request.form.get('max_nb_nodes_in')
mode = request.form.get('mode')
if mode:
mode = 'inter'
else:
mode = 'union'
level = sanitise_level(request.form.get('level'))
## get all selected correlations
filter_types = []
for ob_type in ail_objects.get_all_objects():
correl_option = request.form.get(f'{ob_type}_Check')
if correl_option:
filter_types.append(ob_type)
# list as params
filter_types = ",".join(filter_types)
# redirect to keep history and bookmark
return redirect(url_for('correlation.show_correlation', type=object_type, subtype=subtype, id=obj_id, mode=mode,
max_nodes=max_nodes, level=level, filter=filter_types))
# request.method == 'GET'
else:
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
mode = sanitise_graph_mode(request.args.get('mode'))
level = sanitise_level(request.args.get('level'))
objs_hidden = sanitise_objs_hidden(request.args.get('hidden'))
obj_to_hide = request.args.get('hide')
if obj_to_hide:
objs_hidden.add(obj_to_hide)
related_btc = bool(request.args.get('related_btc', False))
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
# check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
2023-04-05 14:09:06 +00:00
return abort(404)
# object exist
else: # TODO remove old dict key
dict_object = {"type": obj_type,
"id": obj_id,
"object_type": obj_type,
"max_nodes": max_nodes, "mode": mode, "level": level,
"filter": filter_types, "filter_str": ",".join(filter_types),
"hidden": objs_hidden, "hidden_str": ",".join(objs_hidden),
"correlation_id": obj_id,
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id,
options={'tags'}, flask_context=True),
"nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id)
}
if subtype:
dict_object["subtype"] = subtype
dict_object["metadata"]['type_id'] = subtype
else:
dict_object["subtype"] = ''
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
dict_object["metadata_card"]['tags_safe'] = True
2023-04-05 14:09:06 +00:00
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data(),
meta=dict_object["metadata_card"],
ail_tags=dict_object["metadata_card"]["add_tags_modal"])
2019-11-19 08:02:25 +00:00
@correlation.route('/correlation/get/description')
@login_required
@login_read_only
def get_description():
object_id = request.args.get('object_id')
obj_type, subtype, obj_id = ail_objects.get_obj_type_subtype_id_from_global_id(object_id)
# check if obj exist
# # TODO: return error json
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
# object exist
else:
options = {'icon', 'tags', 'tags_safe'}
if obj_type == 'message':
options.add('content')
res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options=options,
flask_context=True)
if 'tags' in res:
res['tags'] = list(res['tags'])
return jsonify(res)
@correlation.route('/correlation/graph_node_json')
@login_required
@login_read_only
def graph_node_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
hidden = request.args.get('hidden')
if hidden:
hidden = set(hidden.split(','))
else:
hidden = set()
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True)
2022-08-19 14:53:31 +00:00
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
return jsonify(json_graph)
@correlation.route('/correlation/delete', methods=['GET'])
@login_required
@login_admin
def correlation_delete():
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id')
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
ail_objects.delete_obj_correlations(obj_type, subtype, obj_id)
return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id))
2023-04-05 14:09:06 +00:00
@correlation.route('/correlation/tags/add', methods=['POST'])
@login_required
2024-09-05 14:40:24 +00:00
@login_admin
2023-04-05 14:09:06 +00:00
def correlation_tags_add():
obj_id = request.form.get('tag_obj_id')
subtype = request.form.get('tag_subtype', '')
obj_type = request.form.get('tag_obj_type')
nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
level = sanitise_level(request.form.get('tag_level'))
2023-04-05 14:09:06 +00:00
filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
hidden = sanitise_objs_hidden(request.form.get('tag_hidden'))
2023-04-05 14:09:06 +00:00
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
# tags
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
if taxonomies_tags or galaxies_tags:
if not Tag.is_valid_tags_taxonomies_galaxy(taxonomies_tags, galaxies_tags):
return {'error': 'Invalid tag(s)'}, 400
tags = taxonomies_tags + galaxies_tags
else:
tags = []
if tags:
ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types,
objs_hidden=hidden,
lvl=level + 1, nb_max=nb_max)
2023-04-05 14:09:06 +00:00
return redirect(url_for('correlation.show_correlation',
type=obj_type, subtype=subtype, id=obj_id,
level=level,
max_nodes=nb_max,
hidden=hidden, hidden_str=",".join(hidden),
2023-04-05 14:09:06 +00:00
filter=",".join(filter_types)))
#####################################################################################
@correlation.route('/relationships/graph_node_json')
@login_required
@login_read_only
def relationships_graph_node_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
return jsonify(json_graph)
@correlation.route('/relationships/chord_graph_json')
@login_required
@login_read_only
def relationships_chord_graph_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationships/chord_mentions_graph_json')
@login_required
@login_read_only
def relationships_chord_mentions_graph_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationship/show', methods=['GET', 'POST'])
@login_required
@login_read_only
def show_relationship():
if request.method == 'POST':
object_type = request.form.get('obj_type')
subtype = request.form.get('subtype')
obj_id = request.form.get('obj_id')
max_nodes = request.form.get('max_nb_nodes_in')
level = sanitise_level(request.form.get('level'))
## get all selected relationships
relationships = []
for relationship in ail_objects.get_relationships():
rel_option = request.form.get(f'relationship_{relationship}_Check')
if rel_option:
relationships.append(relationship)
relationships = ",".join(relationships)
## get all selected objects types
filter_types = []
for ob_type in ail_objects.get_all_objects():
correl_option = request.form.get(f'{ob_type}_Check')
if correl_option:
filter_types.append(ob_type)
# list as params
filter_types = ",".join(filter_types)
# redirect to keep history and bookmark
return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
filter=filter_types, relationships=relationships,
max_nodes=max_nodes, level=level))
# request.method == 'GET'
else:
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
# check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
# object exist
else: # TODO remove old dict key
dict_object = {"type": obj_type,
"id": obj_id,
"object_type": obj_type,
"max_nodes": max_nodes, "level": level,
"correlation_id": obj_id,
"relationships": relationships, "relationships_str": ",".join(relationships),
"filter": filter_types, "filter_str": ",".join(filter_types),
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
"nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
}
if subtype:
dict_object["subtype"] = subtype
dict_object["metadata"]['type_id'] = subtype
else:
dict_object["subtype"] = ''
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
dict_object["metadata_card"]['tags_safe'] = True
return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data(),
meta=dict_object["metadata_card"],
ail_tags=dict_object["metadata_card"]["add_tags_modal"])