From 93ef5418622764083247c8ec1eebfb99a7a9504b Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 29 Nov 2023 16:28:25 +0100 Subject: [PATCH] chg: [chats] add messages threads --- bin/importer/feeders/abstract_chats_feeder.py | 45 ++++- bin/lib/ail_core.py | 4 +- bin/lib/chats_viewer.py | 13 +- bin/lib/correlations_engine.py | 8 +- bin/lib/objects/ChatSubChannels.py | 2 +- bin/lib/objects/ChatThreads.py | 57 +++--- bin/lib/objects/Messages.py | 19 +- bin/lib/objects/abstract_chat_object.py | 4 +- bin/lib/objects/abstract_subtype_object.py | 6 +- bin/lib/objects/ail_objects.py | 6 + var/www/blueprints/chats_explorer.py | 12 ++ .../chats_explorer/ThreadMessages.html | 191 ++++++++++++++++++ .../chats_explorer/block_message.html | 6 + 13 files changed, 329 insertions(+), 44 deletions(-) create mode 100644 var/www/templates/chats_explorer/ThreadMessages.html diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index e82c5e2f..c21e0c78 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN']) from importer.feeders.Default import DefaultFeeder from lib.objects.Chats import Chat from lib.objects import ChatSubChannels +from lib.objects import ChatThreads from lib.objects import Images from lib.objects import Messages from lib.objects import FilesNames @@ -74,13 +75,13 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta']['chat']['id'] def get_subchannel_id(self): - pass + return self.json_data['meta']['chat'].get('subchannel', {}).get('id') def get_subchannels(self): pass def get_thread_id(self): - pass + return self.json_data['meta'].get('thread', {}).get('id') def get_message_id(self): return self.json_data['meta']['id'] @@ -112,7 +113,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta'].get('reply_to') # TODO change to reply ??? def get_message_reply_id(self): - return self.json_data['meta'].get('reply_to', None) + return self.json_data['meta'].get('reply_to', {}).get('message_id') def get_message_content(self): decoded = base64.standard_b64decode(self.json_data['data']) @@ -125,6 +126,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): #### Create Object ID #### chat_id = self.get_chat_id() message_id = self.get_message_id() + thread_id = self.get_thread_id() # channel id # thread id @@ -135,11 +137,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC): self.obj = Images.Image(self.json_data['data-sha256']) else: - obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) + obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, thread_id=thread_id) self.obj = Messages.Message(obj_id) return self.obj - def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads + def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) @@ -170,7 +172,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): chat.add_children(obj_global_id=subchannel.get_global_id()) else: if obj.type == 'message': - chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) + if self.get_thread_id(): + self.process_thread(obj, chat, date, timestamp, reply_id=reply_id) + else: + chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) # if meta.get('subchannels'): # TODO Update icon + names @@ -198,9 +203,32 @@ class AbstractChatFeeder(DefaultFeeder, ABC): subchannel.set_info(meta['info']) if obj.type == 'message': - subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) + if self.get_thread_id(): + self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id) + else: + subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) return subchannel + def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None): + meta = self.json_data['meta']['thread'] + thread_id = self.get_thread_id() + p_chat_id = meta['parent'].get('chat') + p_subchannel_id = meta['parent'].get('subchannel') + p_message_id = meta['parent'].get('message') + + if p_chat_id == self.get_chat_id() and p_subchannel_id == self.get_subchannel_id(): + thread = ChatThreads.create(thread_id, self.get_chat_instance_uuid(), p_chat_id, p_subchannel_id, p_message_id, obj_chat) + thread.add(date, obj) + thread.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) + # TODO OTHERS CORRELATIONS TO ADD + + return thread + + # TODO + # else: + # # ADD NEW MESSAGE REF (used by discord) + + def process_sender(self, new_objs, obj, date, timestamp): meta = self.json_data['meta']['sender'] user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) @@ -298,6 +326,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if media_name: FilesNames.FilesNames().create(media_name, date, message, file_obj=obj) + for reaction in self.get_reactions(): + message.add_reaction(reaction['reaction'], int(reaction['count'])) + for obj in objs: # TODO PERF avoid parsing metas multiple times # CHAT diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 1fd8d97a..3e34e4b8 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -16,8 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None -AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', - 'file-name', 'hhhash', +AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', + 'domain', 'etag', 'favicon', 'file-name', 'hhhash', 'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 76af9f6e..4ddd7065 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ConfigLoader import ConfigLoader from lib.objects import Chats from lib.objects import ChatSubChannels +from lib.objects import ChatThreads from lib.objects import Messages from lib.objects import Usernames @@ -324,7 +325,7 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid): def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): - return {"status": "error", "reason": "Unknown chat"}, 404 + return {"status": "error", "reason": "Unknown subchannel"}, 404 meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) @@ -333,6 +334,16 @@ def api_get_subchannel(chat_id, chat_instance_uuid): meta['messages'], meta['tags_messages'] = subchannel.get_messages() return meta, 200 +def api_get_thread(thread_id, thread_instance_uuid): + thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid) + if not thread.exists(): + return {"status": "error", "reason": "Unknown thread"}, 404 + meta = thread.get_meta({'chat', 'nb_messages'}) + # if meta['chat']: + # meta['chat'] = get_chat_meta_from_global_id(meta['chat']) + meta['messages'], meta['tags_messages'] = thread.get_messages() + return meta, 200 + def api_get_message(message_id): message = Messages.Message(message_id) if not message.exists(): diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index fbf5b75e..6a52caed 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,7 +41,9 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { - "chat": ["image", "user-account"], # message or direct correlation like cve, bitcoin, ... ??? + "chat": ["chat-subchannel", "chat-thread", "image", "user-account"], # message or direct correlation like cve, bitcoin, ... ??? + "chat-subchannel": ["chat", "chat-thread", "image", "message", "user-account"], + "chat-thread": ["chat", "chat-subchannel", "image", "message", "user-account"], # TODO user account "cookie-name": ["domain"], "cryptocurrency": ["domain", "item", "message"], "cve": ["domain", "item", "message"], @@ -53,11 +55,11 @@ CORRELATION_TYPES_BY_OBJ = { "hhhash": ["domain"], "image": ["chat", "message", "user-account"], "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? - "message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ?? + "message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ?? "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "user-account": ["chat", "message"], + "user-account": ["chat", "chat-subchannel", "chat-thread", "message"], "username": ["domain", "item", "message"], # TODO chat-user/account } diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index e3601ce1..f73488a4 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -149,7 +149,7 @@ class ChatSubChannel(AbstractChatObject): class ChatSubChannels(AbstractChatObjects): def __init__(self): - super().__init__('chat-subchannels') + super().__init__('chat-subchannel') # if __name__ == '__main__': # chat = Chat('test', 'telegram') diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index ac78be2a..0790d4ae 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -15,12 +15,8 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id -from lib.data_retention_engine import update_obj_date -from lib.objects import ail_objects -from lib.timeline_engine import Timeline +from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects -from lib.correlations_engine import get_correlation_by_correl_type config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -33,13 +29,13 @@ config_loader = None ################################################################################ ################################################################################ -class Chat(AbstractSubtypeObject): # TODO # ID == username ????? +class ChatThread(AbstractChatObject): """ AIL Chat Object. (strings) """ def __init__(self, id, subtype): - super(Chat, self).__init__('chat-thread', id, subtype) + super().__init__('chat-thread', id, subtype) # def get_ail_2_ail_payload(self): # payload = {'raw': self.get_gzip_content(b64=True), @@ -69,7 +65,7 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? # style = 'fas' # icon = '\uf007' style = 'fas' - icon = '\uf086' + icon = '\uf7a4' return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} def get_meta(self, options=set()): @@ -77,27 +73,42 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) - if 'username': - meta['username'] = self.get_username() + if 'nb_messages': + meta['nb_messages'] = self.get_nb_messages() + # created_at ??? return meta def get_misp_object(self): return - ############################################################################ - ############################################################################ + def create(self, container_obj, message_id): + if message_id: + parent_message = container_obj.get_obj_by_message_id(message_id) + if parent_message: # TODO EXCEPTION IF DON'T EXISTS + self.set_parent(obj_global_id=parent_message) + _, _, parent_id = parent_message.split(':', 2) + self.add_correlation('message', '', parent_id) + else: + self.set_parent(obj_global_id=container_obj.get_global_id()) + self.add_correlation(container_obj.get_type(), container_obj.get_subtype(r_str=True), container_obj.get_id()) - # others optional metas, ... -> # TODO ALL meta in hset +def create(thread_id, chat_instance, chat_id, subchannel_id, message_id, container_obj): + if container_obj.get_type() == 'chat': + new_thread_id = f'{chat_id}/{thread_id}' + # sub-channel + else: + new_thread_id = f'{chat_id}/{subchannel_id}/{thread_id}' - #### Messages #### TODO set parents + thread = ChatThread(new_thread_id, chat_instance) + if not thread.exists(): + thread.create(container_obj, message_id) + return thread - # def get_last_message_id(self): - # - # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') +class ChatThreads(AbstractChatObjects): + def __init__(self): + super().__init__('chat-thread') - - -if __name__ == '__main__': - chat = Chat('test', 'telegram') - r = chat.get_messages() - print(r) +# if __name__ == '__main__': +# chat = Chat('test', 'telegram') +# r = chat.get_messages() +# print(r) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 671e2670..6fb1bb35 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -100,6 +100,13 @@ class Message(AbstractObject): chat_id = self.get_basename().rsplit('_', 1)[0] return chat_id + def get_thread(self): + for child in self.get_childrens(): + obj_type, obj_subtype, obj_id = child.split(':', 2) + if obj_type == 'chat-thread': + nb_messages = r_object.zcard(f'messages:{obj_type}:{obj_subtype}:{obj_id}') + return {'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'nb': nb_messages} + # TODO get Instance ID # TODO get channel ID # TODO get thread ID @@ -245,6 +252,10 @@ class Message(AbstractObject): meta['user-account'] = {'id': 'UNKNOWN'} if 'chat' in options: meta['chat'] = self.get_chat_id() + if 'thread' in options: + thread = self.get_thread() + if thread: + meta['thread'] = thread if 'images' in options: meta['images'] = self.get_images() if 'files-names' in options: @@ -318,10 +329,10 @@ class Message(AbstractObject): def delete(self): pass -def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): +def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): # TODO CHECK COLLISIONS timestamp = int(timestamp) if channel_id and thread_id: - return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ????? + return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}' elif channel_id: return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}' elif thread_id: @@ -329,6 +340,10 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None else: return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}' + # thread id of message + # thread id of chat + # thread id of subchannel + # TODO Check if already exists # def create(source, chat_id, message_id, timestamp, content, tags=[]): def create(obj_id, content, translation=None, tags=[]): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index e58efe99..ee6387d3 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_message_meta(self, message, timestamp=None): # TODO handle file message message = Messages.Message(message[9:]) - meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp) + meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'}, timestamp=timestamp) return meta def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page @@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): tags = {} messages = {} curr_date = None - for message in self._get_messages(nb=50, page=1): + for message in self._get_messages(nb=2000, page=1): timestamp = message[1] date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 9a8aaafc..e02e71e9 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -180,9 +180,9 @@ class AbstractSubtypeObject(AbstractObject, ABC): self.add_correlation('domain', '', domain) # TODO:ADD objects + Stats - def create(self, first_seen, last_seen): - self.set_first_seen(first_seen) - self.set_last_seen(last_seen) + # def create(self, first_seen, last_seen): + # self.set_first_seen(first_seen) + # self.set_last_seen(last_seen) def _delete(self): pass diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 2a9493e9..1b01c2c1 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -14,6 +14,8 @@ from lib import btc_ail from lib import Tag from lib.objects import Chats +from lib.objects import ChatSubChannels +from lib.objects import ChatThreads from lib.objects import CryptoCurrencies from lib.objects import CookiesNames from lib.objects.Cves import Cve @@ -62,6 +64,10 @@ def get_object(obj_type, subtype, obj_id): return Decoded(obj_id) elif obj_type == 'chat': return Chats.Chat(obj_id, subtype) + elif obj_type == 'chat-subchannel': + return ChatSubChannels.ChatSubChannel(obj_id, subtype) + elif obj_type == 'chat-thread': + return ChatThreads.ChatThread(obj_id, subtype) elif obj_type == 'cookie-name': return CookiesNames.CookieName(obj_id) elif obj_type == 'cve': diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 1a6dfba7..925efacd 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -112,6 +112,18 @@ def objects_subchannel_messages(): subchannel = subchannel[0] return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label) +@chats_explorer.route("/chats/explorer/thread", methods=['GET']) +@login_required +@login_read_only +def objects_thread_messages(): + thread_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + thread = chats_viewer.api_get_thread(thread_id, instance_uuid) + if thread[1] != 200: + return create_json_response(thread[0], thread[1]) + else: + meta = thread[0] + return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label) @chats_explorer.route("/objects/message", methods=['GET']) @login_required diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html new file mode 100644 index 00000000..921861c9 --- /dev/null +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -0,0 +1,191 @@ + + + + + Thread Messages - AIL + + + + + + +{# #} + + + + + + + +{# + #} + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+ {{ meta["id"] }} +
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + +
    IDParentFirst seenLast seenNb Messages
    + {{ meta['id'] }} + + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['nb_messages'] }}
    +
    +
    +
    • +{#
  • #} +{#
    #} +{#
    #} +{# Tags:#} +{# {% for tag in meta['tags'] %}#} +{# #} +{# {% endfor %}#} +{# #} +{#
    #} +{#
    • #} +
+ +{# {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}#} +{# {% include 'modals/investigations_register_obj.html' %}#} +{# {% endwith %}#} +{# #} + +
+
+ + {% for tag in meta['tags_messages'] %} + {{ tag }} {{ meta['tags_messages'][tag] }} + {% endfor %} + +
+
+ {% for date in messages %} + {{ date }} + {% endfor %} +
+
+ + + {% include 'objects/image/block_blur_img_slider.html' %} + + +
+
+ + {% for date in meta['messages'] %} + +
+

+ {{ date }} +

+
+ + {% for mess in meta['messages'][date] %} + + {% with message=mess %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %} + + {% endfor %} +
+ {% endfor %} + +
+
+ +
+ +
+
+ + + + + + diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index b06689b9..019de791 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -74,6 +74,12 @@ {% for reaction in message['reactions'] %} {{ reaction }} {{ message['reactions'][reaction] }} {% endfor %} + {% if message['thread'] %} +
+
+ {{ message['thread']['nb'] }} Messages +
+ {% endif %} {% for tag in message['tags'] %} {{ tag }} {% endfor %}