From c1a2bc7eb8b74ef0133e4a110177584c040dbdb4 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 28 May 2024 13:58:36 +0200 Subject: [PATCH] chg: [relationships] messages foraward between chats/user-account + chat chord diagram --- bin/importer/feeders/abstract_chats_feeder.py | 81 ++++-- bin/lib/chats_viewer.py | 49 +++- bin/lib/objects/UsersAccount.py | 10 + bin/lib/objects/ail_objects.py | 6 + bin/lib/relationships_engine.py | 86 ++++++- var/www/blueprints/correlation.py | 13 + .../correlation/show_relationship.html | 236 ++++++++++++++++-- 7 files changed, 431 insertions(+), 50 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 643b65b1..85ca7212 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -128,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta'].get('reply_to', {}).get('message_id') def get_message_forward(self): - return self.json_data['meta'].get('forward') + return self.json_data['meta'].get('forward', {}) def get_message_content(self): decoded = base64.standard_b64decode(self.json_data['data']) @@ -291,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # else: # # ADD NEW MESSAGE REF (used by discord) + ########################################################################################## + + def _process_user(self, meta, date, timestamp, new_objs=None): #################33 timestamp + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + # TODO timeline or/and correlation ???? + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + user_account.update_username_timeline(username.get_global_id(), timestamp) # TODO time.time !!!! (time when meta are retrieved) + + # Username---Message + username.add(date) # TODO # correlation message ??? ############################################################### + + # ADDITIONAL METAS + if meta.get('firstname'): + user_account.set_first_name(meta['firstname']) + if meta.get('lastname'): + user_account.set_last_name(meta['lastname']) + if meta.get('phone'): + user_account.set_phone(meta['phone']) + + if meta.get('icon'): + img = Images.create(meta['icon'], b64=True) + img.add(date, user_account) + user_account.set_icon(img.get_global_id()) + new_objs.add(img) + + if meta.get('info'): + user_account.set_info(meta['info']) + + user_account.add(date) + return user_account + def process_sender(self, new_objs, obj, date, timestamp): meta = self.json_data['meta'].get('sender') if not meta: @@ -339,6 +372,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if self.obj: objs.add(self.obj) new_objs = set() + chats_objs = set() date, timestamp = self.get_message_date_timestamp() @@ -397,14 +431,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC): print(obj.id) # CHAT - chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) + curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) if user_account: # UserAccount---ChatObjects - for obj_chat in chat_objs: + for obj_chat in curr_chats_objs: user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) # if chat: # TODO Chat---Username correlation ??? @@ -416,24 +450,37 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # -> subchannel ? # -> thread id ? + chats_objs.update(curr_chats_objs) ####################################################################### ## FORWARD ## - # chat_fwd = None - # if self.get_json_meta().get('forward'): - # meta_fwd = self.get_message_forward() - # if meta_fwd['chat']: - # chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward') - # - # # TODO chat_fwd -> message - # if chat_fwd: - # for obj in objs: - # if obj.type == 'message': - # chat_fwd.add_relationship(obj.get_global_id(), 'forward') + chat_fwd = None + user_fwd = None + if self.get_json_meta().get('forward'): + meta_fwd = self.get_message_forward() + if meta_fwd.get('chat'): + chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') + if meta_fwd.get('user'): + user_fwd = self._process_user(meta_fwd['user'], date, timestamp, new_objs=new_objs) # TODO date, timestamp ??? + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') + + # TODO chat_fwd -> message + if chat_fwd or user_fwd: + for obj in objs: + if obj.type == 'message': + if chat_fwd: + obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from') + if user_fwd: + obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from') + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + obj.add_relationship(chat_obj.get_global_id(), 'in') # -FORWARD- # return new_objs | objs diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 33fb729a..fa77d108 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -429,7 +429,7 @@ def get_user_account_chats_meta(user_id, chats, subchannels): meta.append(chat_meta) return meta -def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ... +def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ... meta = {} chat = Chats.Chat(chat_id, subtype) chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'}) @@ -515,6 +515,53 @@ def fix_correlations_subchannel_message(): #### API #### +def get_chat_user_account_label(chat_gid): + label = None + obj_type, subtype, obj_id = chat_gid.split(':', 2) + if obj_type == 'chat': + obj = get_obj_chat(obj_type, subtype, obj_id) + username = obj.get_username() + if username: + username = username.split(':', 2)[2] + name = obj.get_name() + if username and name: + label = f'{username} - {name}' + elif username: + label = username + elif name: + label = name + + elif obj_type == 'user-account': + obj = UsersAccount.UserAccount(obj_id, subtype) + username = obj.get_username() + if username: + username = username.split(':', 2)[2] + name = obj.get_name() + if username and name: + label = f'{username} - {name}' + elif username: + label = username + elif name: + label = name + return label + +def enrich_chat_relationships_labels(relationships): + meta = {} + for row in relationships: + if row['source'] not in meta: + label = get_chat_user_account_label(row['source']) + if label: + meta[row['source']] = label + else: + meta[row['source']] = row['source'] + + if row['target'] not in meta: + label = get_chat_user_account_label(row['target']) + if label: + meta[row['target']] = label + else: + meta[row['target']] = row['target'] + return meta def api_get_chat_service_instance(chat_instance_uuid): chat_instance = ChatServiceInstance(chat_instance_uuid) if not chat_instance.exists(): diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f16a487f..a64a5fb0 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject): def get_last_name(self): return self._get_field('lastname') + def get_name(self): + first_name = self.get_first_name() + last_name = self.get_last_name() + if first_name and last_name: + return f'{first_name} {last_name}' + elif first_name: + return first_name + elif last_name: + return last_name + def get_phone(self): return self._get_field('phone') diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 9b73f71e..9461acb9 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -587,6 +587,12 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], fi "links": links, "meta": meta} +def get_chat_relationships_cord_graph(obj_type, subtype, obj_id): + if obj_type == 'chat': + obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) + data = relationships_engine.get_chat_forward_stats(obj_global_id) + return data + return [] # --- RELATIONSHIPS --- # diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py index 511967b3..e166dfaa 100755 --- a/bin/lib/relationships_engine.py +++ b/bin/lib/relationships_engine.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* - +import json import os import sys @@ -16,13 +16,23 @@ config_loader = None RELATIONSHIPS = { - "forward", # forwarded_to + "forwarded_from", + "forwarded_to", # forwarded_to + "in", "mention" } RELATIONSHIPS_OBJS = { - "forward": { - 'chat': {'chat', 'message'}, + "forwarded_from": { + 'chat': {'message'}, + 'message': {'chat', 'user-account'} + }, + "forwarded_to": { + 'chat': {'chat'}, + 'user-account': {'chat'}, + }, + "in": { + 'chat': {'message'}, 'message': {'chat'} }, "mention": {} @@ -51,10 +61,12 @@ def sanityze_obj_types(relationship, obj_type, filter_types): objs_types = get_relationship_objs(relationship, obj_type) if filter_types: filter_types = objs_types.intersection(filter_types) - if not filter_types: + else: filter_types = objs_types - if not filter_types: - return [] + # if not filter_types: + # filter_types = objs_types + # if not filter_types: + # return [] return filter_types # TODO check obj_type @@ -149,3 +161,63 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, # done.add(rel['id']) +# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # + +def get_chat_forward_stats_out(obj_global_id): + nb = {} + for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}): + chat_mess = rel['source'].split('/') + + chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}' + if chat_target not in nb: + nb[chat_target] = 0 + nb[chat_target] += 1 + return nb + +def get_chat_forward_stats_in(obj_global_id): + nb = {} + for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}): + r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'}) + if r: + if not r[0]['target'] in nb: + nb[r[0]['target']] = 0 + nb[r[0]['target']] += 1 + # chat_mess = rel['source'].split('/') + # + # chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}' + # print(chat_target, chat, chat_target == chat) + # if chat is None or chat_target == chat: + # if chat_target not in nb: + # nb[chat_target] = 0 + # nb[chat_target] += 1 + # + # print(json.dumps(nb, indent=4)) + return nb + +def get_chat_forward_stats(obj_global_id): # objs_hidden + data = [] + + # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message'])) + + in_mess = get_chat_forward_stats_out(obj_global_id) + + out_mess = get_chat_forward_stats_in(obj_global_id) + + + # if rel['target'] == obj_global_id: + # # print(rel) + # nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id) + # + for target in in_mess: + data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]}) + + for source in out_mess: + data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]}) + + # print() + # print(in_mess) + # print() + # print(out_mess) + + return data + diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 18a65f87..abb663ab 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects import ail_objects +from lib import chats_viewer from lib import Tag bootstrap_label = Flask_config.bootstrap_label @@ -269,6 +270,18 @@ def relationships_graph_node_json(): json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) return jsonify(json_graph) +@correlation.route('/relationships/chord_graph_json') +@login_required +@login_read_only +def relationships_chord_graph_json(): + obj_id = request.args.get('id') + subtype = request.args.get('subtype') + obj_type = request.args.get('type') + + chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id) + meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph) + + return jsonify({'meta': meta, 'data': chat_json_graph}) @correlation.route('/relationship/show', methods=['GET', 'POST']) @login_required diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index 12889812..044bf134 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -16,7 +16,8 @@ - + +