chg: [relationships] add messages mentions -> chats + user-accounts

This commit is contained in:
terrtia 2024-06-06 11:26:46 +02:00
parent 33bf42538f
commit 883563be56
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
10 changed files with 185 additions and 6 deletions

View file

@ -483,4 +483,41 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
obj.add_relationship(chat_obj.get_global_id(), 'in')
# -FORWARD- #
## MENTION ##
if self.get_json_meta().get('mentions'):
for mention in self.get_json_meta()['mentions'].get('chats', []):
m_obj = self._process_chat(mention, date, new_objs=new_objs)
if m_obj:
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
# TODO PERF
# TODO Keep message obj + chat obj in global var
for obj in objs:
if obj.type == 'message':
obj.add_relationship(m_obj.get_global_id(), 'mention')
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in')
for mention in self.get_json_meta()['mentions'].get('users', []):
m_obj = self._process_user(mention, date, timestamp, new_objs=new_objs) # TODO date, timestamp ???
if m_obj:
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
# TODO PERF
# TODO Keep message obj + chat obj in global var
for obj in objs:
if obj.type == 'message':
obj.add_relationship(m_obj.get_global_id(), 'mention')
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in')
# -MENTION- #
return new_objs | objs

View file

@ -490,6 +490,35 @@ def get_user_account_chats_chord(subtype, user_id):
chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
return chord
def get_user_account_mentions_chord(subtype, user_id):
chord = {'meta': {}, 'data': []}
nb = {}
user_account = UsersAccount.UserAccount(user_id, subtype)
user_account_gid = user_account.get_global_id()
label = get_chat_user_account_label(user_account_gid)
if label:
chord['meta'][user_account_gid] = label
else:
chord['meta'][user_account_gid] = user_account_gid
for mess in user_account.get_messages():
m = Messages.Message(mess[9:])
for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
if rel:
if not rel['target'] in nb:
nb[rel['target']] = 0
nb[rel['target']] += 1
for g_id in nb:
label = get_chat_user_account_label(g_id)
if label:
chord['meta'][g_id] = label
else:
chord['meta'][g_id] = g_id
chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
return chord
def _get_chat_card_meta_options():
return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}

View file

@ -139,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
def get_messages(self):
messages = []
for mess in self.get_correlation('message'):
messages.append(f'message:{mess}')
correl = self.get_correlation('message')
if 'message' in correl:
for mess_id in correl['message']:
messages.append(f'message:{mess_id}')
return messages
def get_messages_by_chat_obj(self, chat_obj):
messages = []
for mess in self.get_correlation_iter_obj(chat_obj, 'message'):

View file

@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
@ -299,6 +299,9 @@ class AbstractObject(ABC):
def get_nb_relationships(self, filter=[]):
return get_obj_nb_relationships(self.get_global_id())
def get_obj_relationships(self, relationships=set(), filter_types=set()):
return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
def add_relationship(self, obj2_global_id, relationship, source=True):
# is source
if source:

View file

@ -594,6 +594,13 @@ def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
return data
return []
def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
if obj_type == 'chat':
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
data = relationships_engine.get_chat_mentions_stats(obj_global_id)
return data
return []
# --- RELATIONSHIPS --- #

View file

@ -22,7 +22,7 @@ RELATIONSHIPS = {
"mention"
}
RELATIONSHIPS_OBJS = {
RELATIONSHIPS_OBJS = { # TODO forward user-account
"forwarded_from": {
'chat': {'message'},
'message': {'chat', 'user-account'}
@ -35,7 +35,11 @@ RELATIONSHIPS_OBJS = {
'chat': {'message'},
'message': {'chat'}
},
"mention": {}
"mention": {
'chat': {'chat', 'user-account', 'message'},
'message': {'chat', 'user-account'},
'user-account': {'chat', 'message'},
},
}
def get_relationships():
@ -221,3 +225,55 @@ def get_chat_forward_stats(obj_global_id): # objs_hidden
return data
##################################################################
def get_chat_mention_stats_in(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
chat_mess = rel['source'].split('/')
chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
if chat_source not in nb:
nb[chat_source] = 0
nb[chat_source] += 1
return nb
def get_chat_mention_stats_out(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
if r:
if not r[0]['target'] in nb:
nb[r[0]['target']] = 0
nb[r[0]['target']] += 1
# chat_mess = rel['source'].split('/')
#
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
# print(chat_target, chat, chat_target == chat)
# if chat is None or chat_target == chat:
# if chat_target not in nb:
# nb[chat_target] = 0
# nb[chat_target] += 1
#
# print(json.dumps(nb, indent=4))
return nb
def get_chat_mentions_stats(obj_global_id): # objs_hidden
data = []
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
out_mess = get_chat_mention_stats_in(obj_global_id)
in_mess = get_chat_mention_stats_out(obj_global_id)
#
for target in in_mess:
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
for source in out_mess:
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
# print()
# print(in_mess)
# print()
# print(out_mess)
return data

View file

@ -306,6 +306,15 @@ def objects_user_account_chats_chord_json():
json_graph = chats_viewer.get_user_account_chats_chord(subtype, user_id)
return jsonify(json_graph)
@chats_explorer.route("/objects/user-account_mentions_chord_json", methods=['GET']) # TODO API
@login_required
@login_read_only
def objects_user_account_mentions_chord_json():
subtype = request.args.get('subtype')
user_id = request.args.get('id')
json_graph = chats_viewer.get_user_account_mentions_chord(subtype, user_id)
return jsonify(json_graph)
@chats_explorer.route("/objects/user-account/chat", methods=['GET'])
@login_required
@login_read_only

View file

@ -283,6 +283,19 @@ def relationships_chord_graph_json():
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationships/chord_mentions_graph_json')
@login_required
@login_read_only
def relationships_chord_mentions_graph_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationship/show', methods=['GET', 'POST'])
@login_required
@login_read_only

View file

@ -50,6 +50,9 @@
<h4>Numbers of Messages Posted by Chat:</h4>
<div id="chord_user_chats" style="max-width: 900px"></div>
<h4>Numbers of Mentions:</h4>
<div id="chord_mentions" style="max-width: 900px"></div>
<h4>User Chats:</h4>
{% for meta_chats in meta['chats'] %}
<div class="my-2">
@ -92,6 +95,10 @@ d3.json(url).then(function(data) {
create_directed_chord_diagram('#chord_user_chats', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
let url2 = "{{ url_for('chats_explorer.objects_user_account_mentions_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
d3.json(url2).then(function(data) {
create_directed_chord_diagram('#chord_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
</script>

View file

@ -248,6 +248,10 @@
<input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="message_Check">Message</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="user-account_Check">User-Account</label>
</div>
</li>
{# <li class="list-group-item text-left">#}
@ -351,8 +355,12 @@
</div>
</div>
<h3>Forwards</h3>
<div id="chart_test" style="max-width: 900px"></div>
<h3>Mentions</h3>
<div id="chart_mentions" style="max-width: 900px"></div>
<table id="table_graph_node_objects">
</table>
@ -381,6 +389,11 @@ $(document).ready(function(){
create_directed_chord_diagram('#chart_test', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
let url2 = "{{ url_for('correlation.relationships_chord_mentions_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
d3.json(url2).then(function(data) {
create_directed_chord_diagram('#chart_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
});
@ -808,6 +821,10 @@ window.all_graph = all_graph;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*
*/
</script>