From 70db33caaf38eec4b86a33d3783afe38816d8d75 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 16 May 2024 15:35:39 +0200 Subject: [PATCH] chg: [relationship] get object relationhips by relationship name + object types --- bin/importer/feeders/abstract_chats_feeder.py | 121 ++++++++--------- bin/lib/objects/ail_objects.py | 20 ++- bin/lib/relationships_engine.py | 126 ++++++++++++------ var/www/blueprints/correlation.py | 30 ++++- .../correlation/show_relationship.html | 28 +++- 5 files changed, 207 insertions(+), 118 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index b48fe4ad..643b65b1 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder) import datetime import os import sys +import time from abc import ABC @@ -164,6 +165,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC): self.obj = Messages.Message(obj_id) return self.obj + # TODO handle subchannel + def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE??? + chat = Chat(meta_chat['id'], self.get_chat_instance_uuid()) + + # Obj Daterange + chat.add(date) + + if meta_chat.get('name'): + chat.set_name(meta_chat['name']) + + if meta_chat.get('info'): + chat.set_info(meta_chat['info']) + + if meta_chat.get('date'): # TODO check if already exists + chat.set_created_at(int(meta_chat['date']['timestamp'])) + + if meta_chat.get('icon'): + img = Images.create(meta_chat['icon'], b64=True) + img.add(date, chat) + chat.set_icon(img.get_global_id()) + if new_objs: + new_objs.add(img) + + if meta_chat.get('username'): + username = Username(meta_chat['username'], self.get_chat_protocol()) + chat.update_username_timeline(username.get_global_id(), int(time.time())) + username.add(date, obj=chat) # TODO TODAY DATE + + return chat + + ############################################################################################################################## + + def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) @@ -365,73 +399,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # CHAT chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) - # # TODO HANDLE OTHERS OBJECT TYPE - # # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # # Message forward + Discussion - # if self.get_json_meta().get('forward'): - # discussion_id = self.get_json_meta().get('discussion') - # forward_from = self.get_message_forward() - # - # if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS - # chat_forward_id = forward_from['from']['id'] - # message_forward_id = forward_from['from']['channel_post'] - # - # # if chat_forward_id == discussion_id: - # # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid()) - # # if linked_chat.exists(): - # # # create thread - # # # add message replies for each childrens - # - # # TODO HANDLE THREAD - # # TODO Change FORWARD META FIELDS - # # meta['forward'] = {} - # # # CHAT ID - # # # SUBCHANNEL ID -> can be None - # # # Message ID - # - # # meta['forward']['origin'] - # # # same as 'forward' - # - # if self.get_json_meta().get('forward'): - # forward = self.get_message_forward() - # f_chat = forward['chat'] - # f_subchannel = forward.get('subchannel') - # f_id = forward.get('id') - # if not f_subchannel: - # chat_forward = Chat(f_chat, self.get_chat_instance_uuid()) - # if chat_forward.exists(): - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') - # # TODO LIST FORWARDED MESSAGES - # - # - # # Discord -> serverID + subchannel ID + message ID - # # Telegram -> chat ID + Message ID - # # + ORIGIN IDs - # - # - # - # # TODO create relationships graph - # - # - # # TODO REMOVE ME - # # Message forward # TODO handle subchannel + message ID - # if self.get_json_meta().get('forward'): - # forward_from = self.get_message_forward() - # print('-----------------------------------------------------------') - # print(forward_from) - # if forward_from: - # forward_from_type = forward_from['from']['type'] - # if forward_from_type == 'channel' or forward_from_type == 'chat': - # chat_forward_id = forward_from['from']['id'] - # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid()) - # if chat_forward.exists(): - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') - # # chat_forward.add_relationship(obj.get_global_id(), 'forward') - # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) @@ -449,4 +416,24 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # -> subchannel ? # -> thread id ? + + ####################################################################### + + ## FORWARD ## + # chat_fwd = None + # if self.get_json_meta().get('forward'): + # meta_fwd = self.get_message_forward() + # if meta_fwd['chat']: + # chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) + # for chat_obj in chat_objs: + # if chat_obj.type == 'chat': + # chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward') + # + # # TODO chat_fwd -> message + # if chat_fwd: + # for obj in objs: + # if obj.type == 'message': + # chat_fwd.add_relationship(obj.get_global_id(), 'forward') + # -FORWARD- # + return new_objs | objs diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 143979ec..9b73f71e 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -562,18 +562,25 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_ # --- CORRELATION --- # +#### RELATIONSHIPS #### + +def get_relationships(): + return relationships_engine.get_relationships() + +def sanitize_relationships(relationships): + return relationships_engine.sanitize_relationships(relationships) def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]): obj = get_object(obj_type, subtype, obj_id) return obj.get_nb_relationships(filter=filter_types) -def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, +def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False): obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) - nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, - filter_types=filter_types, - max_nodes=max_nodes, level=level, - objs_hidden=objs_hidden) + nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships, + filter_types=filter_types, + max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden) # print(meta) meta['objs'] = list(meta['objs']) return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context), @@ -581,6 +588,9 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max "meta": meta} +# --- RELATIONSHIPS --- # + + # if __name__ == '__main__': # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}]) # r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'), diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py index 6791214a..511967b3 100755 --- a/bin/lib/relationships_engine.py +++ b/bin/lib/relationships_engine.py @@ -16,41 +16,87 @@ config_loader = None RELATIONSHIPS = { - "forward", + "forward", # forwarded_to "mention" } + +RELATIONSHIPS_OBJS = { + "forward": { + 'chat': {'chat', 'message'}, + 'message': {'chat'} + }, + "mention": {} +} + def get_relationships(): return RELATIONSHIPS - -def get_obj_relationships_by_type(obj_global_id, relationship): - return r_rel.smembers(f'rel:{relationship}:{obj_global_id}') - -def get_obj_nb_relationships_by_type(obj_global_id, relationship): - return r_rel.scard(f'rel:{relationship}:{obj_global_id}') - -def get_obj_relationships(obj_global_id): - relationships = [] - for relationship in get_relationships(): - for rel in get_obj_relationships_by_type(obj_global_id, relationship): - meta = {'relationship': relationship} - direction, obj_id = rel.split(':', 1) - if direction == 'i': - meta['source'] = obj_id - meta['target'] = obj_global_id - else: - meta['target'] = obj_id - meta['source'] = obj_global_id - - if not obj_id.startswith('chat'): - continue - - meta['id'] = obj_id - # meta['direction'] = direction - relationships.append(meta) +def sanitize_relationships(relationships): + rels = get_relationships() + if relationships: + relationships = set(relationships).intersection(rels) + if not relationships: + relationships = rels + if not relationships: + return [] return relationships -def get_obj_nb_relationships(obj_global_id): +def get_relationship_obj_types(relationship): + return RELATIONSHIPS_OBJS.get(relationship, {}) + +def get_relationship_objs(relationship, obj_type): + return get_relationship_obj_types(relationship).get(obj_type, set()) + +def sanityze_obj_types(relationship, obj_type, filter_types): + objs_types = get_relationship_objs(relationship, obj_type) + if filter_types: + filter_types = objs_types.intersection(filter_types) + if not filter_types: + filter_types = objs_types + if not filter_types: + return [] + return filter_types + +# TODO check obj_type +# TODO sanitize relationships + +def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()): + obj_type = obj_global_id.split(':', 1)[0] + relationships = {} + filter_types = sanityze_obj_types(relationship, obj_type, filter_types) + for o_type in filter_types: + relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}') + return relationships + +def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()): + obj_type = obj_global_id.split(':', 1)[0] + relationships = {} + filter_types = sanityze_obj_types(relationship, obj_type, filter_types) + for o_type in filter_types: + relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}') + return relationships + +def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()): + all_relationships = [] + for relationship in relationships: + obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types) + for obj_type in obj_relationships: + for rel in obj_relationships[obj_type]: + meta = {'relationship': relationship} + direction, obj_id = rel.split(':', 1) + if direction == 'i': + meta['source'] = obj_id + meta['target'] = obj_global_id + else: + meta['target'] = obj_id + meta['source'] = obj_global_id + + meta['id'] = obj_id + # meta['direction'] = direction + all_relationships.append(meta) + return all_relationships + +def get_obj_nb_relationships(obj_global_id): # TODO########################################################################################### nb = {} for relationship in get_relationships(): nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship) @@ -59,27 +105,27 @@ def get_obj_nb_relationships(obj_global_id): # TODO Filter by obj type ??? def add_obj_relationship(source, target, relationship): - r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}') - r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}') - # r_rel.sadd(f'rels:{source}', relationship) - # r_rel.sadd(f'rels:{target}', relationship) + source_type = source.split(':', 1)[0] + target_type = target.split(':', 1)[0] + r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}') + r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}') -def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()): +def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()): links = [] nodes = set() meta = {'complete': True, 'objs': set()} done = set() done_link = set() - _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) return nodes, links, meta -def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()): +def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()): meta['objs'].add(obj_global_id) nodes.add(obj_global_id) - for rel in get_obj_relationships(obj_global_id): + for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types): meta['objs'].add(rel['id']) if rel['id'] in done: @@ -99,13 +145,7 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, if level > 0: next_level = level - 1 - _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) # done.add(rel['id']) - -if __name__ == '__main__': - source = '' - target = '' - add_obj_relationship(source, target, 'forward') - # print(get_obj_relationships(source)) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 983c3f66..18a65f87 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -263,7 +263,10 @@ def relationships_graph_node_json(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) - json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) + relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(',')) + + json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) return jsonify(json_graph) @@ -278,8 +281,28 @@ def show_relationship(): max_nodes = request.form.get('max_nb_nodes_in') level = sanitise_level(request.form.get('level')) + ## get all selected relationships + relationships = [] + for relationship in ail_objects.get_relationships(): + rel_option = request.form.get(f'relationship_{relationship}_Check') + if rel_option: + relationships.append(relationship) + + relationships = ",".join(relationships) + + ## get all selected objects types + filter_types = [] + for ob_type in ail_objects.get_all_objects(): + correl_option = request.form.get(f'{ob_type}_Check') + if correl_option: + filter_types.append(ob_type) + + # list as params + filter_types = ",".join(filter_types) + # redirect to keep history and bookmark return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id, + filter=filter_types, relationships=relationships, max_nodes=max_nodes, level=level)) # request.method == 'GET' @@ -290,6 +313,9 @@ def show_relationship(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True) + relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(',')) + # check if obj_id exist if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) @@ -300,6 +326,8 @@ def show_relationship(): "object_type": obj_type, "max_nodes": max_nodes, "level": level, "correlation_id": obj_id, + "relationships": relationships, "relationships_str": ",".join(relationships), + "filter": filter_types, "filter_str": ",".join(filter_types), "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True), "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id) } diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index 9ad5cc8b..12889812 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -205,13 +205,37 @@