diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py
index 643b65b1..85ca7212 100755
--- a/bin/importer/feeders/abstract_chats_feeder.py
+++ b/bin/importer/feeders/abstract_chats_feeder.py
@@ -128,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta'].get('reply_to', {}).get('message_id')
def get_message_forward(self):
- return self.json_data['meta'].get('forward')
+ return self.json_data['meta'].get('forward', {})
def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data'])
@@ -291,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# else:
# # ADD NEW MESSAGE REF (used by discord)
+ ##########################################################################################
+
+ def _process_user(self, meta, date, timestamp, new_objs=None): #################33 timestamp
+ user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
+ if meta.get('username'):
+ username = Username(meta['username'], self.get_chat_protocol())
+ # TODO timeline or/and correlation ????
+ user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+ user_account.update_username_timeline(username.get_global_id(), timestamp) # TODO time.time !!!! (time when meta are retrieved)
+
+ # Username---Message
+ username.add(date) # TODO # correlation message ??? ###############################################################
+
+ # ADDITIONAL METAS
+ if meta.get('firstname'):
+ user_account.set_first_name(meta['firstname'])
+ if meta.get('lastname'):
+ user_account.set_last_name(meta['lastname'])
+ if meta.get('phone'):
+ user_account.set_phone(meta['phone'])
+
+ if meta.get('icon'):
+ img = Images.create(meta['icon'], b64=True)
+ img.add(date, user_account)
+ user_account.set_icon(img.get_global_id())
+ new_objs.add(img)
+
+ if meta.get('info'):
+ user_account.set_info(meta['info'])
+
+ user_account.add(date)
+ return user_account
+
def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta'].get('sender')
if not meta:
@@ -339,6 +372,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if self.obj:
objs.add(self.obj)
new_objs = set()
+ chats_objs = set()
date, timestamp = self.get_message_date_timestamp()
@@ -397,14 +431,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
print(obj.id)
# CHAT
- chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
+ curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(new_objs, obj, date, timestamp)
if user_account:
# UserAccount---ChatObjects
- for obj_chat in chat_objs:
+ for obj_chat in curr_chats_objs:
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
# if chat: # TODO Chat---Username correlation ???
@@ -416,24 +450,37 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# -> subchannel ?
# -> thread id ?
+ chats_objs.update(curr_chats_objs)
#######################################################################
## FORWARD ##
- # chat_fwd = None
- # if self.get_json_meta().get('forward'):
- # meta_fwd = self.get_message_forward()
- # if meta_fwd['chat']:
- # chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
- # for chat_obj in chat_objs:
- # if chat_obj.type == 'chat':
- # chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward')
- #
- # # TODO chat_fwd -> message
- # if chat_fwd:
- # for obj in objs:
- # if obj.type == 'message':
- # chat_fwd.add_relationship(obj.get_global_id(), 'forward')
+ chat_fwd = None
+ user_fwd = None
+ if self.get_json_meta().get('forward'):
+ meta_fwd = self.get_message_forward()
+ if meta_fwd.get('chat'):
+ chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+ if meta_fwd.get('user'):
+ user_fwd = self._process_user(meta_fwd['user'], date, timestamp, new_objs=new_objs) # TODO date, timestamp ???
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+
+ # TODO chat_fwd -> message
+ if chat_fwd or user_fwd:
+ for obj in objs:
+ if obj.type == 'message':
+ if chat_fwd:
+ obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
+ if user_fwd:
+ obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ obj.add_relationship(chat_obj.get_global_id(), 'in')
# -FORWARD- #
return new_objs | objs
diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py
index 33fb729a..fa77d108 100755
--- a/bin/lib/chats_viewer.py
+++ b/bin/lib/chats_viewer.py
@@ -429,7 +429,7 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
meta.append(chat_meta)
return meta
-def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
+def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
meta = {}
chat = Chats.Chat(chat_id, subtype)
chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
@@ -515,6 +515,53 @@ def fix_correlations_subchannel_message():
#### API ####
+def get_chat_user_account_label(chat_gid):
+ label = None
+ obj_type, subtype, obj_id = chat_gid.split(':', 2)
+ if obj_type == 'chat':
+ obj = get_obj_chat(obj_type, subtype, obj_id)
+ username = obj.get_username()
+ if username:
+ username = username.split(':', 2)[2]
+ name = obj.get_name()
+ if username and name:
+ label = f'{username} - {name}'
+ elif username:
+ label = username
+ elif name:
+ label = name
+
+ elif obj_type == 'user-account':
+ obj = UsersAccount.UserAccount(obj_id, subtype)
+ username = obj.get_username()
+ if username:
+ username = username.split(':', 2)[2]
+ name = obj.get_name()
+ if username and name:
+ label = f'{username} - {name}'
+ elif username:
+ label = username
+ elif name:
+ label = name
+ return label
+
+def enrich_chat_relationships_labels(relationships):
+ meta = {}
+ for row in relationships:
+ if row['source'] not in meta:
+ label = get_chat_user_account_label(row['source'])
+ if label:
+ meta[row['source']] = label
+ else:
+ meta[row['source']] = row['source']
+
+ if row['target'] not in meta:
+ label = get_chat_user_account_label(row['target'])
+ if label:
+ meta[row['target']] = label
+ else:
+ meta[row['target']] = row['target']
+ return meta
def api_get_chat_service_instance(chat_instance_uuid):
chat_instance = ChatServiceInstance(chat_instance_uuid)
if not chat_instance.exists():
diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py
index f16a487f..a64a5fb0 100755
--- a/bin/lib/objects/UsersAccount.py
+++ b/bin/lib/objects/UsersAccount.py
@@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
def get_last_name(self):
return self._get_field('lastname')
+ def get_name(self):
+ first_name = self.get_first_name()
+ last_name = self.get_last_name()
+ if first_name and last_name:
+ return f'{first_name} {last_name}'
+ elif first_name:
+ return first_name
+ elif last_name:
+ return last_name
+
def get_phone(self):
return self._get_field('phone')
diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py
index 9b73f71e..9461acb9 100755
--- a/bin/lib/objects/ail_objects.py
+++ b/bin/lib/objects/ail_objects.py
@@ -587,6 +587,12 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], fi
"links": links,
"meta": meta}
+def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
+ if obj_type == 'chat':
+ obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+ data = relationships_engine.get_chat_forward_stats(obj_global_id)
+ return data
+ return []
# --- RELATIONSHIPS --- #
diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py
index 511967b3..e166dfaa 100755
--- a/bin/lib/relationships_engine.py
+++ b/bin/lib/relationships_engine.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
-
+import json
import os
import sys
@@ -16,13 +16,23 @@ config_loader = None
RELATIONSHIPS = {
- "forward", # forwarded_to
+ "forwarded_from",
+ "forwarded_to", # forwarded_to
+ "in",
"mention"
}
RELATIONSHIPS_OBJS = {
- "forward": {
- 'chat': {'chat', 'message'},
+ "forwarded_from": {
+ 'chat': {'message'},
+ 'message': {'chat', 'user-account'}
+ },
+ "forwarded_to": {
+ 'chat': {'chat'},
+ 'user-account': {'chat'},
+ },
+ "in": {
+ 'chat': {'message'},
'message': {'chat'}
},
"mention": {}
@@ -51,10 +61,12 @@ def sanityze_obj_types(relationship, obj_type, filter_types):
objs_types = get_relationship_objs(relationship, obj_type)
if filter_types:
filter_types = objs_types.intersection(filter_types)
- if not filter_types:
+ else:
filter_types = objs_types
- if not filter_types:
- return []
+ # if not filter_types:
+ # filter_types = objs_types
+ # if not filter_types:
+ # return []
return filter_types
# TODO check obj_type
@@ -149,3 +161,63 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
# done.add(rel['id'])
+# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
+
+def get_chat_forward_stats_out(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
+ chat_mess = rel['source'].split('/')
+
+ chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ if chat_target not in nb:
+ nb[chat_target] = 0
+ nb[chat_target] += 1
+ return nb
+
+def get_chat_forward_stats_in(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+ r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
+ if r:
+ if not r[0]['target'] in nb:
+ nb[r[0]['target']] = 0
+ nb[r[0]['target']] += 1
+ # chat_mess = rel['source'].split('/')
+ #
+ # chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ # print(chat_target, chat, chat_target == chat)
+ # if chat is None or chat_target == chat:
+ # if chat_target not in nb:
+ # nb[chat_target] = 0
+ # nb[chat_target] += 1
+ #
+ # print(json.dumps(nb, indent=4))
+ return nb
+
+def get_chat_forward_stats(obj_global_id): # objs_hidden
+ data = []
+
+ # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+ in_mess = get_chat_forward_stats_out(obj_global_id)
+
+ out_mess = get_chat_forward_stats_in(obj_global_id)
+
+
+ # if rel['target'] == obj_global_id:
+ # # print(rel)
+ # nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
+ #
+ for target in in_mess:
+ data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+ for source in out_mess:
+ data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+ # print()
+ # print(in_mess)
+ # print()
+ # print(out_mess)
+
+ return data
+
diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py
index 18a65f87..abb663ab 100644
--- a/var/www/blueprints/correlation.py
+++ b/var/www/blueprints/correlation.py
@@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.objects import ail_objects
+from lib import chats_viewer
from lib import Tag
bootstrap_label = Flask_config.bootstrap_label
@@ -269,6 +270,18 @@ def relationships_graph_node_json():
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
return jsonify(json_graph)
+@correlation.route('/relationships/chord_graph_json')
+@login_required
+@login_read_only
+def relationships_chord_graph_json():
+ obj_id = request.args.get('id')
+ subtype = request.args.get('subtype')
+ obj_type = request.args.get('type')
+
+ chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
+ meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
+
+ return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationship/show', methods=['GET', 'POST'])
@login_required
diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html
index 12889812..044bf134 100644
--- a/var/www/templates/correlation/show_relationship.html
+++ b/var/www/templates/correlation/show_relationship.html
@@ -16,7 +16,8 @@
-
+
+