From 0dfd92bcd608bdd1d6f0d4354947a73c630a28e8 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 May 2024 15:14:51 +0200 Subject: [PATCH 01/19] chg: [tron] add TRON crytocurrency detection + correlation. Thanks @pventuzelo for the contribution --- bin/lib/ail_core.py | 2 +- bin/lib/objects/CryptoCurrencies.py | 12 ++++++------ bin/modules/Cryptocurrencies.py | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 9483a3d0..5bcb6920 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype if obj_type == 'chat-thread': return r_object.smembers(f'all_chat-thread:subtypes') if obj_type == 'cryptocurrency': - return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] + return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash'] if obj_type == 'pgp': return ['key', 'mail', 'name'] if obj_type == 'username': diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index 01ff0c5e..ac545cc2 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject): pass def is_valid_address(self): - if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin': + if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron': return check_base58_address(self.id) else: return True @@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject): return 'ZEC' elif self.subtype == 'dash': return 'DASH' + elif self.subtype == 'tron': + return 'TRX' return None def get_link(self, flask_context=False): @@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject): def get_all_subtypes(): # return ail_core.get_object_all_subtypes(self.type) - return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] + return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash'] # def build_crypto_regex(subtype, search_id): @@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol): return 'zcash' elif symbol == 'DASH': return 'dash' + elif symbol == 'TRX': + return 'tron' return None @@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype): def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail if subtype == '': pass - elif subtype == 'name': - pass - elif subtype == 'mail': - pass return name_to_search def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False): diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 036b02b4..cf36ed81 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -92,7 +92,13 @@ CURRENCIES = { 'regex': r'\b(? Date: Mon, 13 May 2024 10:51:46 +0200 Subject: [PATCH 02/19] chg: [user message] show user messages by chat --- bin/lib/chats_viewer.py | 55 +++++++- bin/lib/objects/UsersAccount.py | 1 + var/www/blueprints/chats_explorer.py | 20 +++ .../chats_explorer/basic_card_chat.html | 2 +- .../chats_explorer/user_account.html | 2 +- .../chats_explorer/user_chat_messages.html | 133 ++++++++++++++++++ 6 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 var/www/templates/chats_explorer/user_chat_messages.html diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 8e47ef8b..33fb729a 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -328,6 +328,31 @@ def get_username_meta_from_global_id(username_global_id): username = Usernames.Username(username_id, instance_uuid) return username.get_meta(options={'icon'}) +############################################################################### +# TODO Pagination +def list_messages_to_dict(l_messages_id, translation_target=None): + options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'} + meta = {} + curr_date = None + for mess_id in l_messages_id: + message = Messages.Message(mess_id[1:]) + timestamp = message.get_timestamp() + date_day = message.get_date() + date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}' + if date_day != curr_date: + meta[date_day] = [] + curr_date = date_day + meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target) + meta[date_day].append(meta_mess) + + # if mess_dict.get('tags'): + # for tag in mess_dict['tags']: + # if tag not in tags: + # tags[tag] = 0 + # tags[tag] += 1 + # return messages, pagination, tags + return meta + # TODO Filter ## Instance type ## Chats IDS @@ -354,7 +379,7 @@ def get_messages_iterator(filters={}): # threads for threads in chat.get_threads(): thread = ChatThreads.ChatThread(threads['id'], instance_uuid) - _, _ = thread._get_messages(nb=-1) + messages, _ = thread._get_messages(nb=-1) for mess in messages: message_id, _, message_id = mess[0].split(':', ) yield Messages.Message(message_id) @@ -404,6 +429,15 @@ def get_user_account_chats_meta(user_id, chats, subchannels): meta.append(chat_meta) return meta +def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ... + meta = {} + chat = Chats.Chat(chat_id, subtype) + chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'}) + if chat_meta['username']: + chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username']) + + meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None) + return meta def get_user_account_nb_all_week_messages(user_id, chats, subchannels): week = {} @@ -487,12 +521,12 @@ def api_get_chat_service_instance(chat_instance_uuid): return {"status": "error", "reason": "Unknown uuid"}, 404 return chat_instance.get_meta({'chats'}), 200 -def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1): +def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 # print(chat.get_obj_language_stats()) - meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target) + meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -500,7 +534,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa else: if translation_target not in Language.get_translation_languages(): translation_target = None - meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) + if messages: + meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id): @@ -602,6 +637,18 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None): meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels']) return meta, 200 +def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None): + user_account = UsersAccount.UserAccount(user_id, instance_uuid) + if not user_account.exists(): + return {"status": "error", "reason": "Unknown user-account"}, 404 + meta = get_user_account_chat_message(user_id, instance_uuid, chat_id) + meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target) + resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False) + if resp[1] != 200: + return resp + meta['chat'] = resp[0] + return meta, 200 + def api_get_user_account_nb_all_week_messages(user_id, instance_uuid): user_account = UsersAccount.UserAccount(user_id, instance_uuid) if not user_account.exists(): diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f9b09b48..f16a487f 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -157,6 +157,7 @@ class UserAccount(AbstractSubtypeObject): meta['usernames'] = self.get_usernames() if 'icon' in options: meta['icon'] = self.get_icon() + meta['svg_icon'] = meta['icon'] if 'info' in options: meta['info'] = self.get_info() if 'translation' in options and translation_target: diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 6d8d2091..5a1f9b8a 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -297,6 +297,26 @@ def objects_user_account(): ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']), translation_languages=languages, translation_target=target) +@chats_explorer.route("/objects/user-account/chat", methods=['GET']) +@login_required +@login_read_only +def objects_user_account_chat(): + instance_uuid = request.args.get('subtype') + user_id = request.args.get('id') + chat_id = request.args.get('chat_id') + target = request.args.get('target') + if target == "Don't Translate": + target = None + meta = chats_viewer.api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=target) + if meta[1] != 200: + return create_json_response(meta[0], meta[1]) + else: + meta = meta[0] + languages = Language.get_translation_languages() + return render_template('chats_explorer/user_chat_messages.html', meta=meta, bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(meta['user-account']['id'], meta['user-account']['type'], meta['user-account']['subtype']), + translation_languages=languages, translation_target=target) + @chats_explorer.route("objects/user-account/messages/stats/week/all", methods=['GET']) @login_required @login_read_only diff --git a/var/www/templates/chats_explorer/basic_card_chat.html b/var/www/templates/chats_explorer/basic_card_chat.html index d747ff8e..4190788e 100644 --- a/var/www/templates/chats_explorer/basic_card_chat.html +++ b/var/www/templates/chats_explorer/basic_card_chat.html @@ -85,7 +85,7 @@ - {{ meta["nb_messages"] }}   + {{ meta["nb_messages"] }}   {% endif %} diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html index 188779b2..32a61718 100644 --- a/var/www/templates/chats_explorer/user_account.html +++ b/var/www/templates/chats_explorer/user_account.html @@ -45,7 +45,7 @@

User Chats:

{% for meta_chats in meta['chats'] %}
- {% with meta=meta_chats %} + {% with meta=meta_chats,main_obj_id=meta['id'] %} {% include 'chats_explorer/basic_card_chat.html' %} {% endwith %}
diff --git a/var/www/templates/chats_explorer/user_chat_messages.html b/var/www/templates/chats_explorer/user_chat_messages.html new file mode 100644 index 00000000..d79931bc --- /dev/null +++ b/var/www/templates/chats_explorer/user_chat_messages.html @@ -0,0 +1,133 @@ + + + + + User Account - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +

User:

+ {% with meta=meta['user-account'] %} + {% include 'chats_explorer/card_user_account.html' %} + {% endwith %} + +

Chat:

+ {% with meta=meta['chat'] %} + {% include 'chats_explorer/basic_card_chat.html' %} + {% endwith %} + +
+ {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} + {% include 'objects/image/block_blur_img_slider.html' %} +
+ + +
+
+ + {% for date in meta['messages'] %} + +
+

+ {{ date }} +

+
+ + {% for mess in meta['messages'][date] %} + + {% with message=mess %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %} + + {% endfor %} +
+ {% endfor %} + +
+
+ + + +{# {% if meta['chats'] %}#} +{#

User All Messages:

#} +{#
#} +{##} +{#

User Chats:

#} +{# {% for meta_chats in meta['chats'] %}#} +{#
#} +{# {% with meta=meta_chats %}#} +{# {% include 'chats_explorer/basic_card_chat.html' %}#} +{# {% endwith %}#} +{#
#} +{# {% endfor %}#} +{# {% endif %}#} + + +
+ +
+
+ + + + + + + + + + From 4eb1b01370a4d6e7da73409d4ddb1d7940e96aec Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 15 May 2024 10:03:00 +0200 Subject: [PATCH 03/19] chg: [crawler stats] add previous month stats by domain type --- bin/packages/Date.py | 6 ++++++ var/www/blueprints/crawler_splash.py | 11 +++++++++++ .../crawler/crawler_splash/last_crawled.html | 11 +++++++++++ 3 files changed, 28 insertions(+) diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 77d71188..e6c3738b 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -263,3 +263,9 @@ def sanitise_daterange(date_from, date_to, separator='', date_type='str'): date_from = date_to date_to = res return date_from, date_to + +def get_previous_month_date(): + now = datetime.date.today() + first = now.replace(day=1) + last_month = first - datetime.timedelta(days=1) + return last_month.strftime("%Y%m%d") diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 666645cb..15785ebb 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -316,6 +316,17 @@ def crawlers_last_domains_month_json(): stats = crawlers.get_crawlers_stats_by_month(domain_type) return jsonify(stats) +@crawler_splash.route('/crawlers/last/domains/month/previous/json') +@login_required +@login_read_only +def crawlers_last_domains_previous_month_json(): + domain_type = request.args.get('type') + if domain_type not in crawlers.get_crawler_all_types(): + return jsonify({'error': 'Invalid domain type'}), 400 + date = Date.get_previous_month_date() + stats = crawlers.get_crawlers_stats_by_month(domain_type, date=date) + return jsonify(stats) + @crawler_splash.route('/crawlers/last/domains/status/month/json') @login_required @login_read_only diff --git a/var/www/templates/crawler/crawler_splash/last_crawled.html b/var/www/templates/crawler/crawler_splash/last_crawled.html index e26710bf..b553b562 100644 --- a/var/www/templates/crawler/crawler_splash/last_crawled.html +++ b/var/www/templates/crawler/crawler_splash/last_crawled.html @@ -95,6 +95,10 @@
+

Previous Month Stats:

+
+
+ @@ -156,6 +160,13 @@ $(document).ready(function(){ } ); + $.getJSON("{{ url_for('crawler_splash.crawlers_last_domains_previous_month_json') }}?type={{type}}", + function (data) { + let div_width = $("#barchart_type_previous_month").width(); + barchart_stack("barchart_type_previous_month", data, {"width": div_width}); + } + ); + }); function toggle_sidebar(){ From 86f312cbc3ffb6393bb704257d812529031e19c6 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 15 May 2024 10:21:08 +0200 Subject: [PATCH 04/19] chg: [crawler] add function to delete schedules --- bin/lib/crawlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3cecb779..1142163c 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags) return schedule_uuid +def _delete_schedules(): + for schedule_uuid in get_schedulers_uuid(): + schedule = CrawlerSchedule(schedule_uuid) + schedule.delete() + # TODO sanityze UUID def api_delete_schedule(data): schedule_uuid = data.get('uuid') @@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar= external=external) return task_uuid - ## -- CRAWLER TASK -- ## #### CRAWLER TASK API #### From 70db33caaf38eec4b86a33d3783afe38816d8d75 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 16 May 2024 15:35:39 +0200 Subject: [PATCH 05/19] chg: [relationship] get object relationhips by relationship name + object types --- bin/importer/feeders/abstract_chats_feeder.py | 121 ++++++++--------- bin/lib/objects/ail_objects.py | 20 ++- bin/lib/relationships_engine.py | 126 ++++++++++++------ var/www/blueprints/correlation.py | 30 ++++- .../correlation/show_relationship.html | 28 +++- 5 files changed, 207 insertions(+), 118 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index b48fe4ad..643b65b1 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder) import datetime import os import sys +import time from abc import ABC @@ -164,6 +165,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC): self.obj = Messages.Message(obj_id) return self.obj + # TODO handle subchannel + def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE??? + chat = Chat(meta_chat['id'], self.get_chat_instance_uuid()) + + # Obj Daterange + chat.add(date) + + if meta_chat.get('name'): + chat.set_name(meta_chat['name']) + + if meta_chat.get('info'): + chat.set_info(meta_chat['info']) + + if meta_chat.get('date'): # TODO check if already exists + chat.set_created_at(int(meta_chat['date']['timestamp'])) + + if meta_chat.get('icon'): + img = Images.create(meta_chat['icon'], b64=True) + img.add(date, chat) + chat.set_icon(img.get_global_id()) + if new_objs: + new_objs.add(img) + + if meta_chat.get('username'): + username = Username(meta_chat['username'], self.get_chat_protocol()) + chat.update_username_timeline(username.get_global_id(), int(time.time())) + username.add(date, obj=chat) # TODO TODAY DATE + + return chat + + ############################################################################################################################## + + def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) @@ -365,73 +399,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # CHAT chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) - # # TODO HANDLE OTHERS OBJECT TYPE - # # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # # Message forward + Discussion - # if self.get_json_meta().get('forward'): - # discussion_id = self.get_json_meta().get('discussion') - # forward_from = self.get_message_forward() - # - # if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS - # chat_forward_id = forward_from['from']['id'] - # message_forward_id = forward_from['from']['channel_post'] - # - # # if chat_forward_id == discussion_id: - # # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid()) - # # if linked_chat.exists(): - # # # create thread - # # # add message replies for each childrens - # - # # TODO HANDLE THREAD - # # TODO Change FORWARD META FIELDS - # # meta['forward'] = {} - # # # CHAT ID - # # # SUBCHANNEL ID -> can be None - # # # Message ID - # - # # meta['forward']['origin'] - # # # same as 'forward' - # - # if self.get_json_meta().get('forward'): - # forward = self.get_message_forward() - # f_chat = forward['chat'] - # f_subchannel = forward.get('subchannel') - # f_id = forward.get('id') - # if not f_subchannel: - # chat_forward = Chat(f_chat, self.get_chat_instance_uuid()) - # if chat_forward.exists(): - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') - # # TODO LIST FORWARDED MESSAGES - # - # - # # Discord -> serverID + subchannel ID + message ID - # # Telegram -> chat ID + Message ID - # # + ORIGIN IDs - # - # - # - # # TODO create relationships graph - # - # - # # TODO REMOVE ME - # # Message forward # TODO handle subchannel + message ID - # if self.get_json_meta().get('forward'): - # forward_from = self.get_message_forward() - # print('-----------------------------------------------------------') - # print(forward_from) - # if forward_from: - # forward_from_type = forward_from['from']['type'] - # if forward_from_type == 'channel' or forward_from_type == 'chat': - # chat_forward_id = forward_from['from']['id'] - # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid()) - # if chat_forward.exists(): - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') - # # chat_forward.add_relationship(obj.get_global_id(), 'forward') - # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) @@ -449,4 +416,24 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # -> subchannel ? # -> thread id ? + + ####################################################################### + + ## FORWARD ## + # chat_fwd = None + # if self.get_json_meta().get('forward'): + # meta_fwd = self.get_message_forward() + # if meta_fwd['chat']: + # chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) + # for chat_obj in chat_objs: + # if chat_obj.type == 'chat': + # chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward') + # + # # TODO chat_fwd -> message + # if chat_fwd: + # for obj in objs: + # if obj.type == 'message': + # chat_fwd.add_relationship(obj.get_global_id(), 'forward') + # -FORWARD- # + return new_objs | objs diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 143979ec..9b73f71e 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -562,18 +562,25 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_ # --- CORRELATION --- # +#### RELATIONSHIPS #### + +def get_relationships(): + return relationships_engine.get_relationships() + +def sanitize_relationships(relationships): + return relationships_engine.sanitize_relationships(relationships) def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]): obj = get_object(obj_type, subtype, obj_id) return obj.get_nb_relationships(filter=filter_types) -def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, +def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False): obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) - nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, - filter_types=filter_types, - max_nodes=max_nodes, level=level, - objs_hidden=objs_hidden) + nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships, + filter_types=filter_types, + max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden) # print(meta) meta['objs'] = list(meta['objs']) return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context), @@ -581,6 +588,9 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max "meta": meta} +# --- RELATIONSHIPS --- # + + # if __name__ == '__main__': # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}]) # r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'), diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py index 6791214a..511967b3 100755 --- a/bin/lib/relationships_engine.py +++ b/bin/lib/relationships_engine.py @@ -16,41 +16,87 @@ config_loader = None RELATIONSHIPS = { - "forward", + "forward", # forwarded_to "mention" } + +RELATIONSHIPS_OBJS = { + "forward": { + 'chat': {'chat', 'message'}, + 'message': {'chat'} + }, + "mention": {} +} + def get_relationships(): return RELATIONSHIPS - -def get_obj_relationships_by_type(obj_global_id, relationship): - return r_rel.smembers(f'rel:{relationship}:{obj_global_id}') - -def get_obj_nb_relationships_by_type(obj_global_id, relationship): - return r_rel.scard(f'rel:{relationship}:{obj_global_id}') - -def get_obj_relationships(obj_global_id): - relationships = [] - for relationship in get_relationships(): - for rel in get_obj_relationships_by_type(obj_global_id, relationship): - meta = {'relationship': relationship} - direction, obj_id = rel.split(':', 1) - if direction == 'i': - meta['source'] = obj_id - meta['target'] = obj_global_id - else: - meta['target'] = obj_id - meta['source'] = obj_global_id - - if not obj_id.startswith('chat'): - continue - - meta['id'] = obj_id - # meta['direction'] = direction - relationships.append(meta) +def sanitize_relationships(relationships): + rels = get_relationships() + if relationships: + relationships = set(relationships).intersection(rels) + if not relationships: + relationships = rels + if not relationships: + return [] return relationships -def get_obj_nb_relationships(obj_global_id): +def get_relationship_obj_types(relationship): + return RELATIONSHIPS_OBJS.get(relationship, {}) + +def get_relationship_objs(relationship, obj_type): + return get_relationship_obj_types(relationship).get(obj_type, set()) + +def sanityze_obj_types(relationship, obj_type, filter_types): + objs_types = get_relationship_objs(relationship, obj_type) + if filter_types: + filter_types = objs_types.intersection(filter_types) + if not filter_types: + filter_types = objs_types + if not filter_types: + return [] + return filter_types + +# TODO check obj_type +# TODO sanitize relationships + +def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()): + obj_type = obj_global_id.split(':', 1)[0] + relationships = {} + filter_types = sanityze_obj_types(relationship, obj_type, filter_types) + for o_type in filter_types: + relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}') + return relationships + +def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()): + obj_type = obj_global_id.split(':', 1)[0] + relationships = {} + filter_types = sanityze_obj_types(relationship, obj_type, filter_types) + for o_type in filter_types: + relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}') + return relationships + +def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()): + all_relationships = [] + for relationship in relationships: + obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types) + for obj_type in obj_relationships: + for rel in obj_relationships[obj_type]: + meta = {'relationship': relationship} + direction, obj_id = rel.split(':', 1) + if direction == 'i': + meta['source'] = obj_id + meta['target'] = obj_global_id + else: + meta['target'] = obj_id + meta['source'] = obj_global_id + + meta['id'] = obj_id + # meta['direction'] = direction + all_relationships.append(meta) + return all_relationships + +def get_obj_nb_relationships(obj_global_id): # TODO########################################################################################### nb = {} for relationship in get_relationships(): nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship) @@ -59,27 +105,27 @@ def get_obj_nb_relationships(obj_global_id): # TODO Filter by obj type ??? def add_obj_relationship(source, target, relationship): - r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}') - r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}') - # r_rel.sadd(f'rels:{source}', relationship) - # r_rel.sadd(f'rels:{target}', relationship) + source_type = source.split(':', 1)[0] + target_type = target.split(':', 1)[0] + r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}') + r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}') -def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()): +def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()): links = [] nodes = set() meta = {'complete': True, 'objs': set()} done = set() done_link = set() - _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) return nodes, links, meta -def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()): +def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()): meta['objs'].add(obj_global_id) nodes.add(obj_global_id) - for rel in get_obj_relationships(obj_global_id): + for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types): meta['objs'].add(rel['id']) if rel['id'] in done: @@ -99,13 +145,7 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, if level > 0: next_level = level - 1 - _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) # done.add(rel['id']) - -if __name__ == '__main__': - source = '' - target = '' - add_obj_relationship(source, target, 'forward') - # print(get_obj_relationships(source)) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 983c3f66..18a65f87 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -263,7 +263,10 @@ def relationships_graph_node_json(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) - json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) + relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(',')) + + json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) return jsonify(json_graph) @@ -278,8 +281,28 @@ def show_relationship(): max_nodes = request.form.get('max_nb_nodes_in') level = sanitise_level(request.form.get('level')) + ## get all selected relationships + relationships = [] + for relationship in ail_objects.get_relationships(): + rel_option = request.form.get(f'relationship_{relationship}_Check') + if rel_option: + relationships.append(relationship) + + relationships = ",".join(relationships) + + ## get all selected objects types + filter_types = [] + for ob_type in ail_objects.get_all_objects(): + correl_option = request.form.get(f'{ob_type}_Check') + if correl_option: + filter_types.append(ob_type) + + # list as params + filter_types = ",".join(filter_types) + # redirect to keep history and bookmark return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id, + filter=filter_types, relationships=relationships, max_nodes=max_nodes, level=level)) # request.method == 'GET' @@ -290,6 +313,9 @@ def show_relationship(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True) + relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(',')) + # check if obj_id exist if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) @@ -300,6 +326,8 @@ def show_relationship(): "object_type": obj_type, "max_nodes": max_nodes, "level": level, "correlation_id": obj_id, + "relationships": relationships, "relationships_str": ",".join(relationships), + "filter": filter_types, "filter_str": ",".join(filter_types), "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True), "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id) } diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index 9ad5cc8b..12889812 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -205,13 +205,37 @@
    -
  • Relationship
    • +
  • Relationships
    • +
  • +
    + + +
    +
    + + +
    +
    • + +
  • Objects Types
    • + +
  • +
    + + +
    +
    + + +
    +
    • + {#
  • #} {#
    #} {# #} @@ -324,7 +348,7 @@ $(document).ready(function(){ $("#incomplete_graph").hide(); $("#page-Decoded").addClass("active"); - all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}"); + all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}"); all_graph.onResize(); }); From c1a2bc7eb8b74ef0133e4a110177584c040dbdb4 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 28 May 2024 13:58:36 +0200 Subject: [PATCH 06/19] chg: [relationships] messages foraward between chats/user-account + chat chord diagram --- bin/importer/feeders/abstract_chats_feeder.py | 81 ++++-- bin/lib/chats_viewer.py | 49 +++- bin/lib/objects/UsersAccount.py | 10 + bin/lib/objects/ail_objects.py | 6 + bin/lib/relationships_engine.py | 86 ++++++- var/www/blueprints/correlation.py | 13 + .../correlation/show_relationship.html | 236 ++++++++++++++++-- 7 files changed, 431 insertions(+), 50 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 643b65b1..85ca7212 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -128,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta'].get('reply_to', {}).get('message_id') def get_message_forward(self): - return self.json_data['meta'].get('forward') + return self.json_data['meta'].get('forward', {}) def get_message_content(self): decoded = base64.standard_b64decode(self.json_data['data']) @@ -291,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # else: # # ADD NEW MESSAGE REF (used by discord) + ########################################################################################## + + def _process_user(self, meta, date, timestamp, new_objs=None): #################33 timestamp + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + # TODO timeline or/and correlation ???? + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + user_account.update_username_timeline(username.get_global_id(), timestamp) # TODO time.time !!!! (time when meta are retrieved) + + # Username---Message + username.add(date) # TODO # correlation message ??? ############################################################### + + # ADDITIONAL METAS + if meta.get('firstname'): + user_account.set_first_name(meta['firstname']) + if meta.get('lastname'): + user_account.set_last_name(meta['lastname']) + if meta.get('phone'): + user_account.set_phone(meta['phone']) + + if meta.get('icon'): + img = Images.create(meta['icon'], b64=True) + img.add(date, user_account) + user_account.set_icon(img.get_global_id()) + new_objs.add(img) + + if meta.get('info'): + user_account.set_info(meta['info']) + + user_account.add(date) + return user_account + def process_sender(self, new_objs, obj, date, timestamp): meta = self.json_data['meta'].get('sender') if not meta: @@ -339,6 +372,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if self.obj: objs.add(self.obj) new_objs = set() + chats_objs = set() date, timestamp = self.get_message_date_timestamp() @@ -397,14 +431,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC): print(obj.id) # CHAT - chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) + curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) if user_account: # UserAccount---ChatObjects - for obj_chat in chat_objs: + for obj_chat in curr_chats_objs: user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) # if chat: # TODO Chat---Username correlation ??? @@ -416,24 +450,37 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # -> subchannel ? # -> thread id ? + chats_objs.update(curr_chats_objs) ####################################################################### ## FORWARD ## - # chat_fwd = None - # if self.get_json_meta().get('forward'): - # meta_fwd = self.get_message_forward() - # if meta_fwd['chat']: - # chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) - # for chat_obj in chat_objs: - # if chat_obj.type == 'chat': - # chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward') - # - # # TODO chat_fwd -> message - # if chat_fwd: - # for obj in objs: - # if obj.type == 'message': - # chat_fwd.add_relationship(obj.get_global_id(), 'forward') + chat_fwd = None + user_fwd = None + if self.get_json_meta().get('forward'): + meta_fwd = self.get_message_forward() + if meta_fwd.get('chat'): + chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs) + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') + if meta_fwd.get('user'): + user_fwd = self._process_user(meta_fwd['user'], date, timestamp, new_objs=new_objs) # TODO date, timestamp ??? + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') + + # TODO chat_fwd -> message + if chat_fwd or user_fwd: + for obj in objs: + if obj.type == 'message': + if chat_fwd: + obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from') + if user_fwd: + obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from') + for chat_obj in chats_objs: + if chat_obj.type == 'chat': + obj.add_relationship(chat_obj.get_global_id(), 'in') # -FORWARD- # return new_objs | objs diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 33fb729a..fa77d108 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -429,7 +429,7 @@ def get_user_account_chats_meta(user_id, chats, subchannels): meta.append(chat_meta) return meta -def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ... +def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ... meta = {} chat = Chats.Chat(chat_id, subtype) chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'}) @@ -515,6 +515,53 @@ def fix_correlations_subchannel_message(): #### API #### +def get_chat_user_account_label(chat_gid): + label = None + obj_type, subtype, obj_id = chat_gid.split(':', 2) + if obj_type == 'chat': + obj = get_obj_chat(obj_type, subtype, obj_id) + username = obj.get_username() + if username: + username = username.split(':', 2)[2] + name = obj.get_name() + if username and name: + label = f'{username} - {name}' + elif username: + label = username + elif name: + label = name + + elif obj_type == 'user-account': + obj = UsersAccount.UserAccount(obj_id, subtype) + username = obj.get_username() + if username: + username = username.split(':', 2)[2] + name = obj.get_name() + if username and name: + label = f'{username} - {name}' + elif username: + label = username + elif name: + label = name + return label + +def enrich_chat_relationships_labels(relationships): + meta = {} + for row in relationships: + if row['source'] not in meta: + label = get_chat_user_account_label(row['source']) + if label: + meta[row['source']] = label + else: + meta[row['source']] = row['source'] + + if row['target'] not in meta: + label = get_chat_user_account_label(row['target']) + if label: + meta[row['target']] = label + else: + meta[row['target']] = row['target'] + return meta def api_get_chat_service_instance(chat_instance_uuid): chat_instance = ChatServiceInstance(chat_instance_uuid) if not chat_instance.exists(): diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f16a487f..a64a5fb0 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject): def get_last_name(self): return self._get_field('lastname') + def get_name(self): + first_name = self.get_first_name() + last_name = self.get_last_name() + if first_name and last_name: + return f'{first_name} {last_name}' + elif first_name: + return first_name + elif last_name: + return last_name + def get_phone(self): return self._get_field('phone') diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 9b73f71e..9461acb9 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -587,6 +587,12 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], fi "links": links, "meta": meta} +def get_chat_relationships_cord_graph(obj_type, subtype, obj_id): + if obj_type == 'chat': + obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) + data = relationships_engine.get_chat_forward_stats(obj_global_id) + return data + return [] # --- RELATIONSHIPS --- # diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py index 511967b3..e166dfaa 100755 --- a/bin/lib/relationships_engine.py +++ b/bin/lib/relationships_engine.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* - +import json import os import sys @@ -16,13 +16,23 @@ config_loader = None RELATIONSHIPS = { - "forward", # forwarded_to + "forwarded_from", + "forwarded_to", # forwarded_to + "in", "mention" } RELATIONSHIPS_OBJS = { - "forward": { - 'chat': {'chat', 'message'}, + "forwarded_from": { + 'chat': {'message'}, + 'message': {'chat', 'user-account'} + }, + "forwarded_to": { + 'chat': {'chat'}, + 'user-account': {'chat'}, + }, + "in": { + 'chat': {'message'}, 'message': {'chat'} }, "mention": {} @@ -51,10 +61,12 @@ def sanityze_obj_types(relationship, obj_type, filter_types): objs_types = get_relationship_objs(relationship, obj_type) if filter_types: filter_types = objs_types.intersection(filter_types) - if not filter_types: + else: filter_types = objs_types - if not filter_types: - return [] + # if not filter_types: + # filter_types = objs_types + # if not filter_types: + # return [] return filter_types # TODO check obj_type @@ -149,3 +161,63 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, # done.add(rel['id']) +# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # + +def get_chat_forward_stats_out(obj_global_id): + nb = {} + for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}): + chat_mess = rel['source'].split('/') + + chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}' + if chat_target not in nb: + nb[chat_target] = 0 + nb[chat_target] += 1 + return nb + +def get_chat_forward_stats_in(obj_global_id): + nb = {} + for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}): + r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'}) + if r: + if not r[0]['target'] in nb: + nb[r[0]['target']] = 0 + nb[r[0]['target']] += 1 + # chat_mess = rel['source'].split('/') + # + # chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}' + # print(chat_target, chat, chat_target == chat) + # if chat is None or chat_target == chat: + # if chat_target not in nb: + # nb[chat_target] = 0 + # nb[chat_target] += 1 + # + # print(json.dumps(nb, indent=4)) + return nb + +def get_chat_forward_stats(obj_global_id): # objs_hidden + data = [] + + # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message'])) + + in_mess = get_chat_forward_stats_out(obj_global_id) + + out_mess = get_chat_forward_stats_in(obj_global_id) + + + # if rel['target'] == obj_global_id: + # # print(rel) + # nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id) + # + for target in in_mess: + data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]}) + + for source in out_mess: + data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]}) + + # print() + # print(in_mess) + # print() + # print(out_mess) + + return data + diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 18a65f87..abb663ab 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects import ail_objects +from lib import chats_viewer from lib import Tag bootstrap_label = Flask_config.bootstrap_label @@ -269,6 +270,18 @@ def relationships_graph_node_json(): json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) return jsonify(json_graph) +@correlation.route('/relationships/chord_graph_json') +@login_required +@login_read_only +def relationships_chord_graph_json(): + obj_id = request.args.get('id') + subtype = request.args.get('subtype') + obj_type = request.args.get('type') + + chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id) + meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph) + + return jsonify({'meta': meta, 'data': chat_json_graph}) @correlation.route('/relationship/show', methods=['GET', 'POST']) @login_required diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index 12889812..044bf134 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -16,7 +16,8 @@ - + +