chg: [chats] add chats explorer v0

bin/importer/FeederImporter.py

@@ -56,6 +56,8 @@ class FeederImporter(AbstractImporter):
         feeders = [f[:-3] for f in os.listdir(feeder_dir) if os.path.isfile(os.path.join(feeder_dir, f))]
         self.feeders = {}
         for feeder in feeders:
+            if feeder == 'abstract_chats_feeder':
+                continue
             print(feeder)
             part = feeder.split('.')[-1]
             # import json importer class

bin/importer/feeders/Telegram.py

@@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from importer.feeders.Default import DefaultFeeder
+from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
 from lib.ConfigLoader import ConfigLoader
 from lib.objects import ail_objects
 from lib.objects.Chats import Chat
@@ -24,115 +24,15 @@ from lib.objects import UsersAccount
 from lib.objects.Usernames import Username
 
 import base64
-import io
-import gzip
 
-# TODO remove compression ???
+class TelegramFeeder(AbstractChatFeeder):
-def gunzip_bytes_obj(bytes_obj):
-    gunzipped_bytes_obj = None
-    try:
-        in_ = io.BytesIO()
-        in_.write(bytes_obj)
-        in_.seek(0)
-
-        with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
-            gunzipped_bytes_obj = fo.read()
-    except Exception as e:
-        print(f'Global; Invalid Gzip file: {e}')
-
-    return gunzipped_bytes_obj
-
-class TelegramFeeder(DefaultFeeder):
 
     def __init__(self, json_data):
-        super().__init__(json_data)
+        super().__init__('telegram', json_data)
-        self.name = 'telegram'
 
-    def get_obj(self):  # TODO handle others objects -> images, pdf, ...
+    # def get_obj(self):.
-        # Get message date
+    #     obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
-        timestamp = self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
+    #     obj_id = f'message:telegram:{obj_id}'
-        # if self.json_data['meta'].get('date'):
+    #     self.obj = ail_objects.get_obj_from_global_id(obj_id)
-        #     date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
+    #     return self.obj
-        #     date = date.strftime('%Y/%m/%d')
-        # else:
-        #     date = datetime.date.today().strftime("%Y/%m/%d")
-        chat_id = str(self.json_data['meta']['chat']['id'])
-        message_id = str(self.json_data['meta']['id'])
-        obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
-        obj_id = f'message:telegram:{obj_id}'
-        self.obj = ail_objects.get_obj_from_global_id(obj_id)
-        return self.obj
 
-    def process_meta(self):
-        """
-        Process JSON meta field.
-        """
-        # message chat
-        meta = self.json_data['meta']
-        mess_id = self.json_data['meta']['id']
-        if meta.get('reply_to'):
-            reply_to_id = int(meta['reply_to'])
-        else:
-            reply_to_id = None
-
-        timestamp = meta['date']['timestamp']
-        date = datetime.datetime.fromtimestamp(timestamp)
-        date = date.strftime('%Y%m%d')
-
-        if self.json_data.get('translation'):
-            translation = self.json_data['translation']
-        else:
-            translation = None
-        decoded = base64.standard_b64decode(self.json_data['data'])
-        content = gunzip_bytes_obj(decoded)
-        message = Messages.create(self.obj.id, content, translation=translation)
-
-        if meta.get('chat'):
-            chat = Chat(meta['chat']['id'], 'telegram')
-
-            if meta['chat'].get('username'):
-                chat_username = Username(meta['chat']['username'], 'telegram')
-                chat.update_username_timeline(chat_username.get_global_id(), timestamp)
-
-            # Chat---Message
-            chat.add(date)
-            chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
-        else:
-            chat = None
-
-        # message sender
-        if meta.get('sender'):  # TODO handle message channel forward - check if is user
-            user_id = meta['sender']['id']
-            user_account = UsersAccount.UserAccount(user_id, 'telegram')
-            # UserAccount---Message
-            user_account.add(date, obj=message)
-            # UserAccount---Chat
-            user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
-
-            if meta['sender'].get('firstname'):
-                user_account.set_first_name(meta['sender']['firstname'])
-            if meta['sender'].get('lastname'):
-                user_account.set_last_name(meta['sender']['lastname'])
-            if meta['sender'].get('phone'):
-                user_account.set_phone(meta['sender']['phone'])
-
-            if meta['sender'].get('username'):
-                username = Username(meta['sender']['username'], 'telegram')
-                # TODO timeline or/and correlation ????
-                user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
-                user_account.update_username_timeline(username.get_global_id(), timestamp)
-
-                # Username---Message
-                username.add(date)  # TODO # correlation message ???
-
-                # if chat: # TODO Chat---Username correlation ???
-                #     # Chat---Username
-                #     chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
-
-        # if meta.get('fwd_from'):
-            # if meta['fwd_from'].get('post_author') # user first name
-
-        # TODO reply threads ????
-        # message edit ????
-
-        return None

bin/importer/feeders/abstract_chats_feeder.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Abstract Chat JSON Feeder Importer Module
+================
+
+Process Feeder Json (example: Twitter feeder)
+
+"""
+import datetime
+import os
+import sys
+
+from abc import abstractmethod, ABC
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from importer.feeders.Default import DefaultFeeder
+from lib.objects.Chats import Chat
+from lib.objects import ChatSubChannels
+from lib.objects import Messages
+from lib.objects import UsersAccount
+from lib.objects.Usernames import Username
+from lib import chats_viewer
+
+import base64
+import io
+import gzip
+
+# TODO remove compression ???
+def _gunzip_bytes_obj(bytes_obj):
+    gunzipped_bytes_obj = None
+    try:
+        in_ = io.BytesIO()
+        in_.write(bytes_obj)
+        in_.seek(0)
+
+        with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
+            gunzipped_bytes_obj = fo.read()
+    except Exception as e:
+        print(f'Global; Invalid Gzip file: {e}')
+
+    return gunzipped_bytes_obj
+
+class AbstractChatFeeder(DefaultFeeder, ABC):
+
+    def __init__(self, name, json_data):
+        super().__init__(json_data)
+        self.obj = None
+        self.name = name
+
+    def get_chat_protocol(self):  # TODO # # # # # # # # # # # # #
+        return self.name
+
+    def get_chat_network(self):
+        self.json_data['meta'].get('network', None)
+
+    def get_chat_address(self):
+        self.json_data['meta'].get('address', None)
+
+    def get_chat_instance_uuid(self):
+        chat_instance_uuid = chats_viewer.create_chat_service_instance(self.get_chat_protocol(),
+                                                                       network=self.get_chat_network(),
+                                                                       address=self.get_chat_address())
+        # TODO SET
+        return chat_instance_uuid
+
+    def get_chat_id(self):  # TODO RAISE ERROR IF NONE
+        return self.json_data['meta']['chat']['id']
+
+    def get_channel_id(self):
+        pass
+
+    def get_subchannels(self):
+        pass
+
+    def get_thread_id(self):
+        pass
+
+    def get_message_timestamp(self):
+        return self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
+        # if self.json_data['meta'].get('date'):
+        #     date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
+        #     date = date.strftime('%Y/%m/%d')
+        # else:
+        #     date = datetime.date.today().strftime("%Y/%m/%d")
+
+    def get_message_date_timestamp(self):
+        timestamp = self.get_message_timestamp()
+        date = datetime.datetime.fromtimestamp(timestamp)
+        date = date.strftime('%Y%m%d')
+        return date, timestamp
+
+    def get_message_sender_id(self):
+        return self.json_data['meta']['sender']['id']
+
+    def get_message_reply(self):
+        return self.json_data['meta'].get('reply_to')  # TODO change to reply ???
+
+    def get_message_reply_id(self):
+        return self.json_data['meta'].get('reply_to', None)
+
+    def get_message_content(self):
+        decoded = base64.standard_b64decode(self.json_data['data'])
+        return _gunzip_bytes_obj(decoded)
+
+    def get_obj(self):  # TODO handle others objects -> images, pdf, ...
+        #### TIMESTAMP ####
+        timestamp = self.get_message_timestamp()
+
+        #### Create Object ID ####
+        chat_id = str(self.json_data['meta']['chat']['id'])
+        message_id = str(self.json_data['meta']['id'])
+        # channel id
+        # thread id
+
+        obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
+        self.obj = Messages.Message(obj_id)
+        return self.obj
+
+    def process_chat(self, message, date, timestamp, reply_id=None):  # TODO threads
+        meta = self.json_data['meta']['chat']
+        chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
+        chat.add(date)  # TODO ### Dynamic subtype ???
+
+        if meta.get('name'):
+            chat.set_name(meta['name'])
+
+        if meta.get('date'): # TODO check if already exists
+            chat.set_created_at(int(meta['date']['timestamp']))
+
+        if meta.get('username'):
+            username = Username(meta['username'], self.get_chat_protocol())
+            chat.update_username_timeline(username.get_global_id(), timestamp)
+
+        if meta.get('subchannel'):
+            subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id)
+            chat.add_children(obj_global_id=subchannel.get_global_id())
+        else:
+            chat.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
+
+        # if meta.get('subchannels'): # TODO Update icon + names
+
+        return chat
+
+    # def process_subchannels(self):
+    #     pass
+
+    def process_subchannel(self, message, date, timestamp, reply_id=None):  # TODO CREATE DATE
+        meta = self.json_data['meta']['chat']['subchannel']
+        subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid())
+        subchannel.add(date)
+
+        if meta.get('date'): # TODO check if already exists
+            subchannel.set_created_at(int(meta['date']['timestamp']))
+
+        if meta.get('name'):
+            subchannel.set_name(meta['name'])
+            # subchannel.update_name(meta['name'], timestamp) # TODO #################
+
+        subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
+        return subchannel
+
+    def process_sender(self, date, timestamp):
+        meta = self.json_data['meta']['sender']
+        user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
+
+        if meta.get('username'):
+            username = Username(meta['username'], self.get_chat_protocol())
+            # TODO timeline or/and correlation ????
+            user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+            user_account.update_username_timeline(username.get_global_id(), timestamp)
+
+            # Username---Message
+            username.add(date)  # TODO # correlation message ???
+
+        # ADDITIONAL METAS
+        if meta.get('firstname'):
+            user_account.set_first_name(meta['firstname'])
+        if meta.get('lastname'):
+            user_account.set_last_name(meta['lastname'])
+        if meta.get('phone'):
+            user_account.set_phone(meta['phone'])
+
+        return user_account
+
+    # Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
+    # 1) Create/Get MessageInstance                     - # TODO uuidv5 + text like discord and telegram for default
+    # 2) Create/Get CHAT ID                             - Done
+    # 3) Create/Get Channel IF is in channel
+    # 4) Create/Get Thread IF is in thread
+    # 5) Create/Update Username and User-account        - Done
+    def process_meta(self):  # TODO CHECK MANDATORY FIELDS
+        """
+        Process JSON meta filed.
+        """
+        # meta = self.get_json_meta()
+
+        date, timestamp = self.get_message_date_timestamp()
+
+        # REPLY
+        reply_id = self.get_message_reply_id()
+
+        # TODO Translation
+
+        # Content
+        content = self.get_message_content()
+
+        message = Messages.create(self.obj.id, content)  # TODO translation
+
+        # CHAT
+        chat = self.process_chat(message, date, timestamp, reply_id=reply_id)
+
+        # SENDER # TODO HANDLE NULL SENDER
+        user_account = self.process_sender(date, timestamp)
+
+        # UserAccount---Message
+        user_account.add(date, obj=message)
+        # UserAccount---Chat
+        user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
+
+        # if chat: # TODO Chat---Username correlation ???
+        #     # Chat---Username
+        #     chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

bin/lib/ail_core.py

@@ -13,6 +13,7 @@ from lib.ConfigLoader import ConfigLoader
 
 config_loader = ConfigLoader()
 r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
 config_loader = None
 
 AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
@@ -40,9 +41,11 @@ def get_all_objects():
 def get_objects_with_subtypes():
     return ['chat', 'cryptocurrency', 'pgp', 'username']
 
-def get_object_all_subtypes(obj_type):
+def get_object_all_subtypes(obj_type):  # TODO Dynamic subtype
     if obj_type == 'chat':
-        return ['discord', 'jabber', 'telegram']
+        return r_object.smembers(f'all_chat:subtypes')
+    if obj_type == 'chat-subchannel':
+        return r_object.smembers(f'all_chat-subchannel:subtypes')
     if obj_type == 'cryptocurrency':
         return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
     if obj_type == 'pgp':

bin/lib/chats_viewer.py

@@ -0,0 +1,305 @@
+#!/usr/bin/python3
+
+"""
+Chats Viewer
+===================
+
+
+"""
+import os
+import sys
+import time
+import uuid
+
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import Chats
+from lib.objects import ChatSubChannels
+
+config_loader = ConfigLoader()
+r_db = config_loader.get_db_conn("Kvrocks_DB")
+r_crawler = config_loader.get_db_conn("Kvrocks_Crawler")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+
+r_obj = config_loader.get_db_conn("Kvrocks_DB")  # TEMP new DB ????
+
+# # # # # # # #
+#             #
+#   COMMON    #
+#             #
+# # # # # # # #
+
+# TODO ChatDefaultPlatform
+
+# CHAT(type=chat, subtype=platform, id= chat_id)
+
+# Channel(type=channel, subtype=platform, id=channel_id)
+
+# Thread(type=thread, subtype=platform, id=thread_id)
+
+# Message(type=message, subtype=platform, id=message_id)
+
+
+# Protocol/Platform
+
+
+# class ChatProtocols: # TODO Remove Me
+#
+#     def __init__(self):  # name ???? subtype, id ????
+#         # discord, mattermost, ...
+#         pass
+#
+#     def get_chat_protocols(self):
+#         pass
+#
+#     def get_chat_protocol(self, protocol):
+#         pass
+#
+#     ################################################################
+#
+#     def get_instances(self):
+#         pass
+#
+#     def get_chats(self):
+#         pass
+#
+#     def get_chats_by_instance(self, instance):
+#         pass
+#
+#
+# class ChatNetwork:  # uuid or protocol
+#     def __init__(self, network='default'):
+#         self.id = network
+#
+#     def get_addresses(self):
+#         pass
+#
+#
+# class ChatServerAddress: # uuid or protocol + network
+#     def __init__(self, address='default'):
+#         self.id = address
+
+# map uuid -> type + field
+
+# TODO option last protocol/ imported messages/chat -> unread mode ????
+
+# # # # # # # # #
+#               #
+#   PROTOCOLS   #       IRC, discord, mattermost, ...
+#               #
+# # # # # # # # #       TODO icon => UI explorer by protocol + network + instance
+
+def get_chat_protocols():
+    return r_obj.smembers(f'chat:protocols')
+
+def get_chat_protocols_meta():
+    metas = []
+    for protocol_id in get_chat_protocols():
+        protocol = ChatProtocol(protocol_id)
+        metas.append(protocol.get_meta(options={'icon'}))
+    return metas
+
+class ChatProtocol: # TODO first seen last seen ???? + nb by day ????
+    def __init__(self, protocol):
+        self.id = protocol
+
+    def exists(self):
+        return r_db.exists(f'chat:protocol:{self.id}')
+
+    def get_networks(self):
+        return r_db.smembers(f'chat:protocol:{self.id}')
+
+    def get_nb_networks(self):
+        return r_db.scard(f'chat:protocol:{self.id}')
+
+    def get_icon(self):
+        if self.id == 'discord':
+            icon = {'style': 'fab', 'icon': 'fa-discord'}
+        elif self.id == 'telegram':
+            icon = {'style': 'fab', 'icon': 'fa-telegram'}
+        else:
+            icon = {}
+        return icon
+
+    def get_meta(self, options=set()):
+        meta = {'id': self.id}
+        if 'icon' in options:
+            meta['icon'] = self.get_icon()
+        return meta
+
+    # def get_addresses(self):
+    #     pass
+    #
+    # def get_instances_uuids(self):
+    #     pass
+
+
+# # # # # # # # # # # # # #
+#                         #
+#   ChatServiceInstance   #
+#                         #
+# # # # # # # # # # # # # #
+
+# uuid -> protocol + network + server
+class ChatServiceInstance:
+    def __init__(self, instance_uuid):
+        self.uuid = instance_uuid
+
+    def exists(self):
+        return r_obj.exists(f'chatSerIns:{self.uuid}')
+
+    def get_protocol(self): # return objects ????
+        return r_obj.hget(f'chatSerIns:{self.uuid}', 'protocol')
+
+    def get_network(self): # return objects ????
+        network = r_obj.hget(f'chatSerIns:{self.uuid}', 'network')
+        if network:
+            return network
+
+    def get_address(self): # return objects ????
+        address = r_obj.hget(f'chatSerIns:{self.uuid}', 'address')
+        if address:
+            return address
+
+    def get_meta(self, options=set()):
+        meta = {'uuid': self.uuid,
+                'protocol': self.get_protocol(),
+                'network': self.get_network(),
+                'address': self.get_address()}
+        if 'chats' in options:
+            meta['chats'] = []
+            for chat_id in self.get_chats():
+                meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'}))
+        return meta
+
+    def get_nb_chats(self):
+        return Chats.Chats().get_nb_ids_by_subtype(self.uuid)
+
+    def get_chats(self):
+        return Chats.Chats().get_ids_by_subtype(self.uuid)
+
+def get_chat_service_instances():
+    return r_obj.smembers(f'chatSerIns:all')
+
+def get_chat_service_instance_uuid(protocol, network, address):
+    if not network:
+        network = ''
+    if not address:
+        address = ''
+    return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address)
+
+def get_chat_service_instance(protocol, network, address):
+    instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
+    if instance_uuid:
+        return ChatServiceInstance(instance_uuid)
+
+def create_chat_service_instance(protocol, network=None, address=None):
+    instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
+    if instance_uuid:
+        return instance_uuid
+    else:
+        if not network:
+            network = ''
+        if not address:
+            address = ''
+        instance_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f'{protocol}|{network}|{address}'))
+        r_obj.sadd(f'chatSerIns:all', instance_uuid)
+
+        # map instance - uuid
+        r_obj.hset(f'map:chatSerIns:{protocol}:{network}', address, instance_uuid)
+
+        r_obj.hset(f'chatSerIns:{instance_uuid}', 'protocol', protocol)
+        if network:
+            r_obj.hset(f'chatSerIns:{instance_uuid}', 'network', network)
+        if address:
+            r_obj.hset(f'chatSerIns:{instance_uuid}', 'address', address)
+
+        # protocols
+        r_obj.sadd(f'chat:protocols', protocol)  # TODO first seen / last seen
+
+        # protocol -> network
+        r_obj.sadd(f'chat:protocol:networks:{protocol}', network)
+
+        return instance_uuid
+
+
+
+
+    # INSTANCE ===> CHAT IDS
+
+
+
+
+
+    # protocol -> instance_uuids => for protocol->networks -> protocol+network => HGETALL
+    # protocol+network -> instance_uuids => HGETALL
+
+    # protocol -> networks  ???default??? or ''
+
+    # --------------------------------------------------------
+    # protocol+network -> addresses => HKEYS
+    # protocol+network+addresse => HGET
+
+
+# Chat -> subtype=uuid, id = chat id
+
+
+# instance_uuid -> chat id
+
+
+# protocol - uniq ID
+# protocol + network -> uuid ????
+# protocol + network + address -> uuid
+
+#######################################################################################
+
+def get_subchannels_meta_from_global_id(subchannels):
+    meta = []
+    for sub in subchannels:
+        _, instance_uuid, sub_id = sub.split(':', 2)
+        subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
+        meta.append(subchannel.get_meta({'nb_messages'}))
+    return meta
+
+def api_get_chat_service_instance(chat_instance_uuid):
+    chat_instance = ChatServiceInstance(chat_instance_uuid)
+    if not chat_instance.exists():
+        return {"status": "error", "reason": "Unknown uuid"}, 404
+    return chat_instance.get_meta({'chats'}), 200
+
+def api_get_chat(chat_id, chat_instance_uuid):
+    chat = Chats.Chat(chat_id, chat_instance_uuid)
+    if not chat.exists():
+        return {"status": "error", "reason": "Unknown chat"}, 404
+    meta = chat.get_meta({'img', 'subchannels', 'username'})
+    if meta['subchannels']:
+        print(meta['subchannels'])
+        meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
+    return meta, 200
+
+def api_get_subchannel(chat_id, chat_instance_uuid):
+    subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
+    if not subchannel.exists():
+        return {"status": "error", "reason": "Unknown chat"}, 404
+    meta = subchannel.get_meta({'img', 'nb_messages'})
+    meta['messages'], meta['tags_messages'] = subchannel.get_messages()
+    return meta, 200
+
+# # # # # # # # # # LATER
+#                 #
+#   ChatCategory  #
+#                 #
+# # # # # # # # # #
+
+
+if __name__ == '__main__':
+    r = get_chat_service_instances()
+    print(r)
+    r = ChatServiceInstance(r.pop())
+    print(r.get_meta({'chats'}))
+    # r = get_chat_protocols()
+    # print(r)

bin/lib/objects/ChatSubChannels.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from datetime import datetime
+
+from flask import url_for
+# from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
+
+from lib.data_retention_engine import update_obj_date
+from lib.objects import ail_objects
+from lib.timeline_engine import Timeline
+
+from lib.correlations_engine import get_correlation_by_correl_type
+
+config_loader = ConfigLoader()
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class ChatSubChannel(AbstractChatObject):
+    """
+    AIL Chat Object. (strings)
+    """
+
+    # ID -> <CHAT ID>/<SubChannel ID>  subtype = chat_instance_uuid
+    def __init__(self, id, subtype):
+        super(ChatSubChannel, self).__init__('chat-subchannel', id, subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):  # TODO
+        # if self.subtype == 'telegram':
+        #     style = 'fab'
+        #     icon = '\uf2c6'
+        # elif self.subtype == 'discord':
+        #     style = 'fab'
+        #     icon = '\uf099'
+        # else:
+        #     style = 'fas'
+        #     icon = '\uf007'
+        style = 'fas'
+        icon = '\uf086'
+        return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
+
+    # TODO TIME LAST MESSAGES
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['tags'] = self.get_tags(r_list=True)
+        meta['name'] = self.get_name()
+        if 'img' in options:
+            meta['sub'] = self.get_img()
+        if 'nb_messages':
+            meta['nb_messages'] = self.get_nb_messages()
+        return meta
+
+    def get_misp_object(self):
+        # obj_attrs = []
+        # if self.subtype == 'telegram':
+        #     obj = MISPObject('telegram-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('username', value=self.id))
+        #
+        # elif self.subtype == 'twitter':
+        #     obj = MISPObject('twitter-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('name', value=self.id))
+        #
+        # else:
+        #     obj = MISPObject('user-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('username', value=self.id))
+        #
+        # first_seen = self.get_first_seen()
+        # last_seen = self.get_last_seen()
+        # if first_seen:
+        #     obj.first_seen = first_seen
+        # if last_seen:
+        #     obj.last_seen = last_seen
+        # if not first_seen or not last_seen:
+        #     self.logger.warning(
+        #         f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
+        #
+        # for obj_attr in obj_attrs:
+        #     for tag in self.get_tags():
+        #         obj_attr.add_tag(tag)
+        # return obj
+        return
+
+    ############################################################################
+    ############################################################################
+
+    # others optional metas, ... -> # TODO ALL meta in hset
+
+    def _get_timeline_name(self):
+        return Timeline(self.get_global_id(), 'username')
+
+    def update_name(self, name, timestamp):
+        self._get_timeline_name().add_timestamp(timestamp, name)
+
+
+    # TODO # # # # # # # # # # #
+    def get_users(self):
+        pass
+
+    #### Categories ####
+
+    #### Threads ####
+
+    #### Messages #### TODO set parents
+
+    # def get_last_message_id(self):
+    #
+    #     return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
+
+
+class ChatSubChannels(AbstractChatObjects):
+    def __init__(self):
+        super().__init__('chat-subchannels')
+
+# if __name__ == '__main__':
+#     chat = Chat('test', 'telegram')
+#     r = chat.get_messages()
+#     print(r)

bin/lib/objects/ChatThreads.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from datetime import datetime
+
+from flask import url_for
+# from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+from lib.data_retention_engine import update_obj_date
+from lib.objects import ail_objects
+from lib.timeline_engine import Timeline
+
+from lib.correlations_engine import get_correlation_by_correl_type
+
+config_loader = ConfigLoader()
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
+    """
+    AIL Chat Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(Chat, self).__init__('chat-thread', id, subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):  # TODO
+        # if self.subtype == 'telegram':
+        #     style = 'fab'
+        #     icon = '\uf2c6'
+        # elif self.subtype == 'discord':
+        #     style = 'fab'
+        #     icon = '\uf099'
+        # else:
+        #     style = 'fas'
+        #     icon = '\uf007'
+        style = 'fas'
+        icon = '\uf086'
+        return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['id'] = self.id
+        meta['subtype'] = self.subtype
+        meta['tags'] = self.get_tags(r_list=True)
+        if 'username':
+            meta['username'] = self.get_username()
+        return meta
+
+    def get_misp_object(self):
+        return
+
+    ############################################################################
+    ############################################################################
+
+    # others optional metas, ... -> # TODO ALL meta in hset
+
+    #### Messages #### TODO set parents
+
+    # def get_last_message_id(self):
+    #
+    #     return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
+
+
+
+if __name__ == '__main__':
+    chat = Chat('test', 'telegram')
+    r = chat.get_messages()
+    print(r)

bin/lib/objects/Chats.py

@@ -15,6 +15,9 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib import ail_core
 from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
+
+
 from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 from lib.data_retention_engine import update_obj_date
 from lib.objects import ail_objects
@@ -33,19 +36,14 @@ config_loader = None
 ################################################################################
 ################################################################################
 
-class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
+class Chat(AbstractChatObject):
     """
-    AIL Chat Object. (strings)
+    AIL Chat Object.
     """
 
     def __init__(self, id, subtype):
         super(Chat, self).__init__('chat', id, subtype)
 
-    # def get_ail_2_ail_payload(self):
-    #     payload = {'raw': self.get_gzip_content(b64=True),
-    #                 'compress': 'gzip'}
-    #     return payload
-
     # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
     def delete(self):
         # # TODO:
@@ -74,9 +72,16 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
 
     def get_meta(self, options=set()):
         meta = self._get_meta(options=options)
-        meta['id'] = self.id
+        meta['name'] = self.get_name()
-        meta['subtype'] = self.subtype
         meta['tags'] = self.get_tags(r_list=True)
+        if 'img':
+            meta['icon'] = self.get_img()
+        if 'username' in options:
+            meta['username'] = self.get_username()
+        if 'subchannels' in options:
+            meta['subchannels'] = self.get_subchannels()
+        if 'nb_subchannels':
+            meta['nb_subchannels'] = self.get_nb_subchannels()
         return meta
 
     def get_misp_object(self):
@@ -112,11 +117,6 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
     ############################################################################
     ############################################################################
 
-    # others optional metas, ... -> # TODO ALL meta in hset
-
-    def get_name(self):  # get username ????
-        pass
-
     # users that send at least a message else participants/spectator
     # correlation created by messages
     def get_users(self):
@@ -138,22 +138,22 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
     def update_username_timeline(self, username_global_id, timestamp):
         self._get_timeline_username().add_timestamp(timestamp, username_global_id)
 
+    #### ChatSubChannels ####
+
+
+    #### Categories ####
+
+    #### Threads ####
+
+    #### Messages #### TODO set parents
 
     # def get_last_message_id(self):
     #
     #     return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
 
-    def get_obj_message_id(self, obj_id):
-        if obj_id.endswith('.gz'):
-            obj_id = obj_id[:-3]
-        return int(obj_id.split('_')[-1])
-
     def _get_message_timestamp(self, obj_global_id):
         return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
 
-    def _get_messages(self):
-        return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
-
     def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
         obj = ail_objects.get_obj_from_global_id(obj_global_id)
         mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
@@ -179,26 +179,6 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
         mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
         return mess_dict
 
-
-    def get_messages(self, start=0, page=1, nb=500):  # TODO limit nb returned, # TODO add replies
-        start = 0
-        stop = -1
-        # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}')
-
-        # TODO chat without username ???? -> chat ID ????
-
-        messages = {}
-        curr_date = None
-        for message in self._get_messages():
-            date = datetime.fromtimestamp(message[1])
-            date_day = date.strftime('%Y/%m/%d')
-            if date_day != curr_date:
-                messages[date_day] = []
-                curr_date = date_day
-            mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)
-            messages[date_day].append(mess_dict)
-        return messages
-
         # Zset with ID ???  id -> item id ??? multiple id == media + text
         #                   id -> media id
         # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
@@ -229,43 +209,12 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
     #         domain = get_item_domain(item_id)
     #         self.add_correlation('domain', '', domain)
 
-    # TODO kvrocks exception if key don't exists
-    def get_obj_by_message_id(self, mess_id):
-        return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id)
-
     # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn
-    def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None):
-        r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id)
-        r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp})
 
-        if reply_id:
-            reply_obj = self.get_obj_by_message_id(reply_id)
-            if reply_obj:
-                self.add_obj_children(reply_obj, obj_global_id)
-            else:
-                self.add_message_cached_reply(reply_id, mess_id)
-
-        # ADD cached replies
-        for reply_obj in self.get_cached_message_reply(mess_id):
-            self.add_obj_children(obj_global_id, reply_obj)
-
-    def _get_message_cached_reply(self, message_id):
-        return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
-
-    def get_cached_message_reply(self, message_id):
-        objs_global_id = []
-        for mess_id in self._get_message_cached_reply(message_id):
-            obj_global_id = self.get_obj_by_message_id(mess_id)
-            if obj_global_id:
-                objs_global_id.append(obj_global_id)
-        return objs_global_id
-
-    def add_message_cached_reply(self, reply_to_id, message_id):
-        r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id)
-        r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600)
-
-    # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering
 
+class Chats(AbstractChatObjects):
+    def __init__(self):
+        super().__init__('chat')
 
 # TODO factorize
 def get_all_subtypes():
@@ -280,28 +229,6 @@ def get_all():
 def get_all_by_subtype(subtype):
     return get_all_id('chat', subtype)
 
-# # TODO FILTER NAME + Key + mail
-# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
-#
-#     return name_to_search
-#
-# def search_usernames_by_name(name_to_search, subtype, r_pos=False):
-#     usernames = {}
-#     # for subtype in subtypes:
-#     r_name = sanitize_username_name_to_search(name_to_search, subtype)
-#     if not name_to_search or isinstance(r_name, dict):
-#         # break
-#         return usernames
-#     r_name = re.compile(r_name)
-#     for user_name in get_all_usernames_by_subtype(subtype):
-#         res = re.search(r_name, user_name)
-#         if res:
-#             usernames[user_name] = {}
-#             if r_pos:
-#                 usernames[user_name]['hl-start'] = res.start()
-#                 usernames[user_name]['hl-end'] = res.end()
-#     return usernames
-
 
 if __name__ == '__main__':
     chat = Chat('test', 'telegram')

bin/lib/objects/Messages.py

@@ -43,6 +43,10 @@ config_loader = None
 # /!\ handle null chat and message id -> chat = uuid and message = timestamp ???
 
 
+# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<message ID> => telegram without channels
+# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<message ID>
+# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Thread ID>/<message ID>
+# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<Thread ID>/<message ID>
 class Message(AbstractObject):
     """
     AIL Message Object. (strings)
@@ -86,7 +90,7 @@ class Message(AbstractObject):
         return dirs[-2]
 
     def get_message_id(self):  # TODO optimize
-        message_id = self.get_basename().rsplit('_', 1)[1]
+        message_id = self.get_basename().rsplit('/', 1)[1]
         # if message_id.endswith('.gz'):
         #     message_id = message_id[:-3]
         return message_id
@@ -97,6 +101,10 @@ class Message(AbstractObject):
         #     chat_id = chat_id[:-3]
         return chat_id
 
+    # TODO get Instance ID
+    # TODO get channel ID
+    # TODO get thread  ID
+
     def get_user_account(self):
         user_account = self.get_correlation('user-account')
         if user_account.get('user-account'):
@@ -255,8 +263,16 @@ class Message(AbstractObject):
     def delete(self):
         pass
 
-def create_obj_id(source, chat_id, message_id, timestamp):
+def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None):
-    return f'{source}/{timestamp}/{chat_id}_{message_id}'
+    timestamp = int(timestamp)
+    if channel_id and thread_id:
+        return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}'  # TODO add thread ID ?????
+    elif channel_id:
+        return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
+    elif thread_id:
+        return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}'
+    else:
+        return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
 
 # TODO Check if already exists
 # def create(source, chat_id, message_id, timestamp, content, tags=[]):

bin/lib/objects/abstract_chat_object.py

@@ -0,0 +1,241 @@
+# -*-coding:UTF-8 -*
+"""
+Base Class for AIL Objects
+"""
+
+##################################
+# Import External packages
+##################################
+import os
+import sys
+from abc import ABC
+
+from datetime import datetime
+# from flask import url_for
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject
+from lib.ail_core import get_object_all_subtypes, zscan_iter ################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.Messages import Message
+from lib.objects.UsersAccount import UserAccount
+from lib.objects.Usernames import Username
+from lib.data_retention_engine import update_obj_date
+
+from packages import Date
+
+# LOAD CONFIG
+config_loader = ConfigLoader()
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+config_loader = None
+
+# # FIXME: SAVE SUBTYPE NAMES ?????
+
+class AbstractChatObject(AbstractSubtypeObject, ABC):
+    """
+    Abstract Subtype Object
+    """
+
+    def __init__(self, obj_type, id, subtype):
+        """ Abstract for all the AIL object
+
+        :param obj_type: object type (item, ...)
+        :param id: Object ID
+        """
+        super().__init__(obj_type, id, subtype)
+
+    # get useraccount / username
+    # get users ?
+    # timeline name ????
+    # info
+    # created
+    # last imported/updated
+
+    # TODO get instance
+    # TODO get protocol
+    # TODO get network
+    # TODO get address
+
+    def get_chat(self):  # require ail object TODO ##
+        if self.type != 'chat':
+            parent = self.get_parent()
+            obj_type, _ = parent.split(':', 1)
+            if obj_type == 'chat':
+                return parent
+
+    def get_subchannels(self):
+        subchannels = []
+        if self.type == 'chat':  # category ???
+            print(self.get_childrens())
+            for obj_global_id in self.get_childrens():
+                print(obj_global_id)
+                obj_type, _ = obj_global_id.split(':', 1)
+                if obj_type == 'chat-subchannel':
+                    subchannels.append(obj_global_id)
+        return subchannels
+
+    def get_nb_subchannels(self):
+        nb = 0
+        if self.type == 'chat':
+            for obj_global_id in self.get_childrens():
+                obj_type, _ = obj_global_id.split(':', 1)
+                if obj_type == 'chat-subchannel':
+                    nb += 1
+        return nb
+
+    def get_threads(self):
+        threads = []
+        for obj_global_id in self.get_childrens():
+            obj_type, _ = obj_global_id.split(':', 1)
+            if obj_type == 'chat-thread':
+                threads.append(obj_global_id)
+        return threads
+
+    def get_created_at(self):
+        return self._get_field('created_at')
+
+    def set_created_at(self, timestamp):
+        self._set_field('created_at', timestamp)
+
+    def get_name(self):
+        name = self._get_field('name')
+        if not name:
+            name = ''
+        return name
+
+    def set_name(self, name):
+        self._set_field('name', name)
+
+    def get_img(self):
+        return self._get_field('img')
+
+    def set_img(self, icon):
+        self._set_field('img', icon)
+
+    def get_nb_messages(self):
+        return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
+
+    def _get_messages(self):  # TODO paginate
+        return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
+
+    def get_message_meta(self, message, parent=True, mess_datetime=None):  # TODO handle file message
+        obj = Message(message[9:])
+        mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
+        print(mess_dict)
+        if mess_dict.get('parent') and parent:
+            mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
+        if mess_dict.get('user-account'):
+            _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3)
+            print(mess_dict['user-account'])
+            user_account = UserAccount(user_account_id, user_account_subtype)
+            mess_dict['user-account'] = {}
+            mess_dict['user-account']['type'] = user_account.get_type()
+            mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
+            mess_dict['user-account']['id'] = user_account.get_id()
+            username = user_account.get_username()
+            if username:
+                _, username_account_subtype, username_account_id = username.split(':', 3)
+                username = Username(username_account_id, username_account_subtype).get_default_meta(link=False)
+            mess_dict['user-account']['username'] = username  # TODO get username at the given timestamp ???
+        else:
+            mess_dict['user-account'] = {'id': 'UNKNOWN'}
+
+        if not mess_datetime:
+            obj_mess_id = message.get_timestamp()
+            mess_datetime = datetime.fromtimestamp(obj_mess_id)
+        mess_dict['date'] = mess_datetime.isoformat(' ')
+        mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
+        return mess_dict
+
+    def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ????
+        # TODO return message meta
+        tags = {}
+        messages = {}
+        curr_date = None
+        for message in self._get_messages():
+            date = datetime.fromtimestamp(message[1])
+            date_day = date.strftime('%Y/%m/%d')
+            if date_day != curr_date:
+                messages[date_day] = []
+                curr_date = date_day
+            mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)  # TODO use object
+            messages[date_day].append(mess_dict)
+
+            if mess_dict.get('tags'):
+                for tag in mess_dict['tags']:
+                    if tag not in tags:
+                        tags[tag] = 0
+                    tags[tag] += 1
+        return messages, tags
+
+    # TODO REWRITE ADD OR ADD MESSAGE ????
+    # add
+    # add message
+
+    def get_obj_by_message_id(self, message_id):
+        return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id)
+
+    def add_message_cached_reply(self, reply_id, message_id):
+        r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id)
+        r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600)
+
+    def _get_message_cached_reply(self, message_id):
+        return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
+
+    def get_cached_message_reply(self, message_id):
+        objs_global_id = []
+        for mess_id in self._get_message_cached_reply(message_id):
+            obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION
+            if obj_global_id:
+                objs_global_id.append(obj_global_id)
+        return objs_global_id
+
+    def add_message(self, obj_global_id, message_id, timestamp, reply_id=None):
+        r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id)
+        r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)})
+
+        # MESSAGE REPLY
+        if reply_id:
+            reply_obj = self.get_obj_by_message_id(reply_id)  # TODO CATCH EXCEPTION
+            if reply_obj:
+                self.add_obj_children(reply_obj, obj_global_id)
+            else:
+                self.add_message_cached_reply(reply_id, message_id)
+
+
+    # get_messages_meta ????
+
+# TODO move me to abstract subtype
+class AbstractChatObjects(ABC):
+    def __init__(self, type):
+        self.type = type
+
+    def add_subtype(self, subtype):
+        r_object.sadd(f'all_{self.type}:subtypes', subtype)
+
+    def get_subtypes(self):
+        return r_object.smembers(f'all_{self.type}:subtypes')
+
+    def get_nb_ids_by_subtype(self, subtype):
+        return r_object.zcard(f'{self.type}_all:{subtype}')
+
+    def get_ids_by_subtype(self, subtype):
+        print(subtype)
+        print(f'{self.type}_all:{subtype}')
+        return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
+
+    def get_all_id_iterator_iter(self, subtype):
+        return zscan_iter(r_object, f'{self.type}_all:{subtype}')
+
+    def get_ids(self):
+        pass
+
+    def search(self):
+        pass
+
+
+

bin/lib/objects/abstract_object.py

@@ -289,23 +289,24 @@ class AbstractObject(ABC):
     def get_parent(self):
         return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent')
 
-    def get_children(self):
+    def get_childrens(self):
         return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
 
-    def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
+    def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None):  # TODO # REMOVE ITEM DUP
         if not obj_global_id:
             if obj_subtype is None:
                 obj_subtype = ''
             obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
         r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id)
 
-    def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
+    def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP
         if not obj_global_id:
             if obj_subtype is None:
                 obj_subtype = ''
             obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
         r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id)
 
+    ## others objects ##
     def add_obj_children(self, parent_global_id, son_global_id):
         r_object.sadd(f'child:{parent_global_id}', son_global_id)
         r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id)

bin/lib/objects/abstract_subtype_object.py

@@ -88,7 +88,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
     def _get_meta(self, options=None):
         if options is None:
             options = set()
-        meta = {'first_seen': self.get_first_seen(),
+        meta = {'id': self.id,
+                'subtype': self.subtype,
+                'first_seen': self.get_first_seen(),
                 'last_seen': self.get_last_seen(),
                 'nb_seen': self.get_nb_seen()}
         if 'icon' in options:
@@ -150,8 +152,11 @@ class AbstractSubtypeObject(AbstractObject, ABC):
 #               => data Retention + efficient search
 #
 #
+    def _add_subtype(self):
+        r_object.sadd(f'all_{self.type}:subtypes', self.subtype)
 
     def add(self, date, obj=None):
+        self._add_subtype()
         self.update_daterange(date)
         update_obj_date(date, self.type, self.subtype)
         # daily

var/www/Flask_server.py

@@ -50,7 +50,7 @@ from blueprints.objects_title import objects_title
 from blueprints.objects_cookie_name import objects_cookie_name
 from blueprints.objects_etag import objects_etag
 from blueprints.objects_hhhash import objects_hhhash
-from blueprints.objects_chat import objects_chat
+from blueprints.chats_explorer import chats_explorer
 
 Flask_dir = os.environ['AIL_FLASK']
 
@@ -108,7 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl)
 app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
 app.register_blueprint(objects_etag, url_prefix=baseUrl)
 app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
-app.register_blueprint(objects_chat, url_prefix=baseUrl)
+app.register_blueprint(chats_explorer, url_prefix=baseUrl)
 
 # =========       =========#
 

var/www/blueprints/chats_explorer.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import chats_viewer
+
+
+
+############################################
+
+from lib import ail_core
+from lib.objects import ail_objects
+from lib import chats_viewer
+from lib.objects import Chats
+from lib.objects import ChatSubChannels
+
+# ============ BLUEPRINT ============
+chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
+# ============ FUNCTIONS ============
+
+# ============= ROUTES ==============
+
+@chats_explorer.route("/chats/explorer", methods=['GET'])
+@login_required
+@login_read_only
+def chats_explorer_dashboard():
+    return
+
+@chats_explorer.route("chats/explorer/protocols", methods=['GET'])
+@login_required
+@login_read_only
+def chats_explorer_protocols():
+    protocols = chats_viewer.get_chat_protocols_meta()
+    return render_template('chats_protocols.html', protocols=protocols)
+
+@chats_explorer.route("chats/explorer/instance", methods=['GET'])
+@login_required
+@login_read_only
+def chats_explorer_instance():
+    intance_uuid = request.args.get('uuid')
+    chat_instance = chats_viewer.api_get_chat_service_instance(intance_uuid)
+    if chat_instance[1] != 200:
+        return create_json_response(chat_instance[0], chat_instance[1])
+    else:
+        chat_instance = chat_instance[0]
+        return render_template('chat_instance.html', chat_instance=chat_instance)
+
+@chats_explorer.route("chats/explorer/chat", methods=['GET'])
+@login_required
+@login_read_only
+def chats_explorer_chat():
+    chat_id = request.args.get('id')
+    instance_uuid = request.args.get('uuid')
+    chat = chats_viewer.api_get_chat(chat_id, instance_uuid)
+    if chat[1] != 200:
+        return create_json_response(chat[0], chat[1])
+    else:
+        chat = chat[0]
+        return render_template('chat_viewer.html', chat=chat)
+
+@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
+@login_required
+@login_read_only
+def objects_subchannel_messages():
+    subchannel_id = request.args.get('id')
+    instance_uuid = request.args.get('uuid')
+    subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid)
+    if subchannel[1] != 200:
+        return create_json_response(subchannel[0], subchannel[1])
+    else:
+        subchannel = subchannel[0]
+        return render_template('SubChannelMessages.html', subchannel=subchannel)
+
+@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
+@login_required
+@login_read_only
+def objects_message():
+    message_id = request.args.get('id')
+    message = chats_viewer.api_get_message(message_id)
+    if message[1] != 200:
+        return create_json_response(message[0], message[1])
+    else:
+        message = message[0]
+        return render_template('ChatMessage.html', message=message)
+
+#############################################################################################
+#############################################################################################
+#############################################################################################
+
+
+@chats_explorer.route("/objects/chat/messages", methods=['GET'])
+@login_required
+@login_read_only
+def objects_dashboard_chat():
+    chat = request.args.get('id')
+    subtype = request.args.get('subtype')
+    chat = Chats.Chat(chat, subtype)
+    if chat.exists():
+        messages, mess_tags = chat.get_messages()
+        print(messages)
+        print(chat.get_subchannels())
+        meta = chat.get_meta({'icon', 'username'})
+        if meta.get('username'):
+            meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta()
+        print(meta)
+        return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label)
+    else:
+        return abort(404)

var/www/blueprints/objects_chat.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-'''
-    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
-'''
-
-import os
-import sys
-import json
-
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
-from flask_login import login_required, current_user
-
-# Import Role_Manager
-from Role_Manager import login_admin, login_analyst, login_read_only
-
-sys.path.append(os.environ['AIL_BIN'])
-##################################
-# Import Project packages
-##################################
-from lib import ail_core
-from lib.objects import abstract_subtype_object
-from lib.objects import ail_objects
-from lib.objects import Chats
-from packages import Date
-
-# ============ BLUEPRINT ============
-objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat'))
-
-# ============ VARIABLES ============
-bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
-
-def create_json_response(data, status_code):
-    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
-
-# ============ FUNCTIONS ============
-
-# ============= ROUTES ==============
-
-
-@objects_chat.route("/objects/chat/messages", methods=['GET'])
-@login_required
-@login_read_only
-def objects_dashboard_chat():
-    chat = request.args.get('id')
-    subtype = request.args.get('subtype')
-    chat = Chats.Chat(chat, subtype)
-    if chat.exists():
-        messages = chat.get_messages()
-        meta = chat.get_meta({'icon'})
-        print(meta)
-        return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label)
-    else:
-        return abort(404)
-
-
-

var/www/templates/chats_explorer/ChatMessage.html

@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chat Messages - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+	<style>
+        .chat-message-left,
+        .chat-message-right {
+            display: flex;
+            flex-shrink: 0;
+        }
+        .chat-message-right {
+            flex-direction: row-reverse;
+            margin-left: auto
+        }
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="card my-3">
+                    <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+                        <h3 class="text-secondary">{{ meta["id"] }} :</h3>
+                        <ul class="list-group mb-2">
+                            <li class="list-group-item py-0">
+                                <div class="row">
+                                    <div class="col-md-10">
+                                        <table class="table">
+                                            <thead>
+                                                <tr>
+                                                    <th>Object subtype</th>
+                                                    <th>First seen</th>
+                                                    <th>Last seen</th>
+                                                    <th>Username</th>
+                                                    <th>Nb seen</th>
+                                                </tr>
+                                            </thead>
+                                            <tbody>
+                                                <tr>
+                                                    <td>
+                                                        <svg height="26" width="26">
+                                                            <g class="nodes">
+                                                                <circle cx="13" cy="13" r="13" fill="{{ meta["icon"]["color"] }}"></circle>
+                                                                <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta["icon"]["style"] }}" font-size="16px">{{ meta["icon"]["icon"] }}</text>
+                                                            </g>
+                                                        </svg>
+                                                        {{ meta["subtype"] }}
+                                                    </td>
+                                                    <td>{{ meta['first_seen'] }}</td>
+                                                    <td>{{ meta['last_seen'] }}</td>
+                                                    <td>
+                                                        {% if 'username' in meta %}
+                                                            {{ meta['username']['id'] }}
+                                                        {% endif %}
+                                                    </td>
+                                                    <td>{{ meta['nb_seen'] }}</td>
+                                                </tr>
+                                            </tbody>
+                                        </table>
+                                    </div>
+                                    <div class="col-md-1">
+                                        <div id="sparkline"></div>
+                                    </div>
+                                </div>
+                            </li>
+                            <li class="list-group-item py-0">
+                                <br>
+                                <div class="mb-3">
+                                    Tags:
+                                    {% for tag in meta['tags'] %}
+                                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
+                                        data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">
+                                            {{ tag }}
+                                        </button>
+                                    {%  endfor %}
+                                    <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+                                        <i class="far fa-plus-square"></i>
+                                    </button>
+                                </div>
+                            </li>
+                        </ul>
+
+                        {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}
+                            {% include 'modals/investigations_register_obj.html' %}
+                        {% endwith %}
+                        <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+                            <i class="fas fa-microscope"></i> Investigations
+                        </button>
+
+                    </div>
+                </div>
+
+                {% for tag in mess_tags %}
+                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
+                {%  endfor %}
+
+                <div>
+                    <div class="list-group d-inline-block">
+                        {% for date in messages %}
+                            <a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
+                        {% endfor %}
+                    </div>
+                </div>
+
+                <div class="position-relative">
+                    <div class="chat-messages p-4">
+
+                        <h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
+
+                        <div class="chat-message-left pb-1">
+                            <div>
+                                <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
+                                <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
+                            </div>
+                            <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
+                                <div class="font-weight-bold mb-1">
+                                    {% if mess['user-account']['username'] %}
+                                        {{ mess['user-account']['username']['id'] }}
+                                    {% else %}
+                                        {{ mess['user-account']['id'] }}
+                                    {% endif %}
+                                </div>
+                                {% if mess['reply_to'] %}
+                                    <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
+                                        <div class="font-weight-bold mb-1">
+                                            {% if mess['reply_to']['user-account']['username'] %}
+                                                {{ mess['reply_to']['user-account']['username']['id'] }}
+                                            {% else %}
+                                                {{ mess['reply_to']['user-account']['id'] }}
+                                            {% endif %}
+                                        </div>
+                                        <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
+                                        {% for tag in mess['reply_to']['tags'] %}
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        {%  endfor %}
+                                        <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+{#                                                <div class="">#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
+{#                                                </div>#}
+                                    </div>
+                                {% endif %}
+                                <pre class="my-0">{{ mess['content'] }}</pre>
+                                {% for tag in mess['tags'] %}
+                                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                {%  endfor %}
+                                <div class="">
+                                    <a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
+                                    <a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
+                                </div>
+                            </div>
+                        </div>
+
+                    </div>
+                </div>
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/chats_explorer/ChatMessages.html

@@ -58,6 +58,7 @@
                                                     <th>Object subtype</th>
                                                     <th>First seen</th>
                                                     <th>Last seen</th>
+                                                    <th>Username</th>
                                                     <th>Nb seen</th>
                                                 </tr>
                                             </thead>
@@ -74,6 +75,11 @@
                                                     </td>
                                                     <td>{{ meta['first_seen'] }}</td>
                                                     <td>{{ meta['last_seen'] }}</td>
+                                                    <td>
+                                                        {% if 'username' in meta %}
+                                                            {{ meta['username']['id'] }}
+                                                        {% endif %}
+                                                    </td>
                                                     <td>{{ meta['nb_seen'] }}</td>
                                                 </tr>
                                             </tbody>
@@ -111,11 +117,23 @@
                     </div>
                 </div>
 
+                {% for tag in mess_tags %}
+                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
+                {%  endfor %}
+
+                <div>
+                    <div class="list-group d-inline-block">
+                        {% for date in messages %}
+                            <a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
+                        {% endfor %}
+                    </div>
+                </div>
+
                 <div class="position-relative">
                     <div class="chat-messages p-4">
 
                         {% for date in messages %}
-                            <h2><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
+                            <h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
                             {% for mess in messages[date] %}
 
                                 <div class="chat-message-left pb-1">

var/www/templates/chats_explorer/SubChannelMessages.html

@@ -0,0 +1,211 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Sub-Channel Messages - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+{#	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+{#	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+	<style>
+        .chat-message-left,
+        .chat-message-right {
+            display: flex;
+            flex-shrink: 0;
+        }
+        .chat-message-right {
+            flex-direction: row-reverse;
+            margin-left: auto
+        }
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="card my-3">
+                    <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+                        <h3 class="text-secondary">{{ subchannel["id"] }} :</h3>
+                        <ul class="list-group mb-2">
+                            <li class="list-group-item py-0">
+                                <div class="row">
+                                    <div class="col-md-10">
+                                        <table class="table">
+                                            <thead>
+                                                <tr>
+                                                    <th>Chat Instance</th>
+                                                    <th>First seen</th>
+                                                    <th>Last seen</th>
+                                                    <th>Username</th>
+                                                    <th>Nb Messages</th>
+                                                </tr>
+                                            </thead>
+                                            <tbody>
+                                                <tr>
+                                                    <td>
+                                                        {{ subchannel["subtype"] }}
+                                                    </td>
+                                                    <td>{{ subchannel['first_seen'] }}</td>
+                                                    <td>{{ subchannel['last_seen'] }}</td>
+                                                    <td>
+                                                        {% if 'username' in subchannel %}
+                                                            {{ subchannel['username'] }}
+                                                        {% endif %}
+                                                    </td>
+                                                    <td>{{ subchannel['nb_messages'] }}</td>
+                                                </tr>
+                                            </tbody>
+                                        </table>
+                                    </div>
+                                </div>
+                            </li>
+                            <li class="list-group-item py-0">
+                                <br>
+                                <div class="mb-3">
+                                    Tags:
+                                    {% for tag in subchannel['tags'] %}
+                                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
+                                        data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ subchannel["subtype"] }}" data-objid="{{ subchannel["id"] }}">
+                                            {{ tag }}
+                                        </button>
+                                    {%  endfor %}
+                                    <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+                                        <i class="far fa-plus-square"></i>
+                                    </button>
+                                </div>
+                            </li>
+                        </ul>
+
+                        {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}
+                            {% include 'modals/investigations_register_obj.html' %}
+                        {% endwith %}
+                        <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+                            <i class="fas fa-microscope"></i> Investigations
+                        </button>
+
+                    </div>
+                </div>
+
+                {% for tag in mess_tags %}
+                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
+                {%  endfor %}
+
+                <div>
+                    <div class="list-group d-inline-block">
+                        {% for date in messages %}
+                            <a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
+                        {% endfor %}
+                    </div>
+                </div>
+
+                <div class="position-relative">
+                    <div class="chat-messages p-4">
+
+                        {% for date in subchannel['messages'] %}
+                            <h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
+                            {% for mess in subchannel['messages'][date] %}
+
+                                <div class="chat-message-left pb-1">
+                                    <div>
+                                        <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
+                                        <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
+                                    </div>
+                                    <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
+                                        <div class="font-weight-bold mb-1">
+                                            {% if mess['user-account']['username'] %}
+                                            	{{ mess['user-account']['username']['id'] }}
+                                            {% else %}
+                                            	{{ mess['user-account']['id'] }}
+                                            {% endif %}
+                                        </div>
+                                        {% if mess['reply_to'] %}
+                                            <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
+                                                <div class="font-weight-bold mb-1">
+                                                    {% if mess['reply_to']['user-account']['username'] %}
+                                                        {{ mess['reply_to']['user-account']['username']['id'] }}
+                                                    {% else %}
+                                                        {{ mess['reply_to']['user-account']['id'] }}
+                                                    {% endif %}
+                                                </div>
+                                                <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
+                                                {% for tag in mess['reply_to']['tags'] %}
+                                                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                                {%  endfor %}
+                                                <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+{#                                                <div class="">#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
+{#                                                </div>#}
+                                            </div>
+                                        {% endif %}
+                                        <pre class="my-0">{{ mess['content'] }}</pre>
+                                        {% for tag in mess['tags'] %}
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        {%  endfor %}
+                                        <div class="">
+                                            <a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
+                                            <a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
+                                        </div>
+                                    </div>
+                                </div>
+
+                            {% endfor %}
+                            <br>
+                        {% endfor %}
+
+                    </div>
+                </div>
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/chats_explorer/chat_instance.html

@@ -0,0 +1,123 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chats Protocols - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="card my-3">
+
+                    <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+                        <h4 class="text-secondary">{{ chat_instance["protocol"] }} :</h4>
+                        <ul class="list-group mb-2">
+                            <li class="list-group-item py-0">
+                                <table class="table">
+                                    <thead>
+                                    <tr>
+                                        <th>Chat Instance</th>
+                                        <th>Network</th>
+                                        <th>Address</th>
+                                        <th>Nb Chats</th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    <tr>
+                                        <td>
+                                            {{ chat_instance["uuid"] }}
+                                        </td>
+                                        <td>{% if chat_instance["network"] %}{{ chat_instance["network"]}}{% endif %}</td>
+                                        <td>{% if chat_instance["address"] %}{{ chat_instance["address"] }}{% endif %}</td>
+                                        <td>{{ chat_instance["chats"] | length }}</td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                            </li>
+                        </ul>
+
+                    </div>
+                </div>
+
+                <table id="tablechats" class="table table-striped table-bordered">
+                    <thead class="bg-dark text-white">
+                    <tr>
+                        <th>Icon</th>
+                        <th>Name</th>
+                        <th>ID</th>
+                        <th>First Seen</th>
+                        <th>Last Seen</th>
+                        <th>NB Chats</th>
+                    </tr>
+                    </thead>
+                    <tbody style="font-size: 15px;">
+                    {% for chat in chat_instance["chats"] %}
+                        <tr>
+                            <td></td>
+                            <td>{{ chat['name'] }}</td>
+                            <td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>
+                            <td>{{ chat['first_seen'] }}</td>
+							<td>{{ chat['last_seen'] }}</td>
+                            <td>{{ chat['nb_subchannels'] }}</td>
+                        </tr>
+                    {% endfor %}
+                    </tbody>
+                </table>
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_cve").addClass("active");
+
+        $('#tablechats').DataTable({
+            "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 4, "desc" ]]
+        });
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/chats_explorer/chat_viewer.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chats Protocols - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="card my-3">
+
+                    <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+                        <h4 class="text-secondary">{{ chat["username"] }} {{ chat["id"] }} :</h4>
+                        <ul class="list-group mb-2">
+                            <li class="list-group-item py-0">
+                                <table class="table">
+                                    <thead class="bg-dark text-white">
+                                    <tr>
+                                        <th>Icon</th>
+                                        <th>Name</th>
+                                        <th>ID</th>
+                                        <th>First Seen</th>
+                                        <th>Last Seen</th>
+                                        <th>NB Sub-Channels</th>
+                                    </tr>
+                                    </thead>
+                                    <tbody style="font-size: 15px;">
+                                    <tr>
+                                            <td></td>
+                                            <td>{{ chat['name'] }}</td>
+                                            <td>{{ chat['id']  }}</td>
+                                            <td>{{ chat['first_seen'] }}</td>
+                                            <td>{{ chat['last_seen'] }}</td>
+                                            <td>{{ chat['nb_subchannels'] }}</td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                            </li>
+                        </ul>
+
+                    </div>
+                </div>
+
+                {% if chat['subchannels'] %}
+                    <h4>Sub-Channels:</h4>
+                    <table id="tablesubchannels" class="table">
+                        <thead class="bg-dark text-white">
+                        <tr>
+                            <th>Icon</th>
+                            <th>Name</th>
+                            <th>ID</th>
+                            <th>First Seen</th>
+                            <th>Last Seen</th>
+                            <th>NB Messages</th>
+                        </tr>
+                        </thead>
+                        <tbody style="font-size: 15px;">
+                        {% for meta in chat["subchannels"] %}
+                            <tr>
+                                <td>
+                                   <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
+                                </td>
+                                <td><b>{{ meta['name'] }}</b></td>
+                                <td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
+                                <td>{{ meta['first_seen'] }}</td>
+                                <td>{{ meta['last_seen'] }}</td>
+                                <td>{{ meta['nb_messages'] }}</td>
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+
+                {% endif %}
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_cve").addClass("active");
+
+        {% if chat['subchannels'] %}
+            $('#tablesubchannels').DataTable({
+                "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+                "iDisplayLength": 10,
+                "order": [[ 4, "desc" ]]
+            });
+        {% endif %}
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/chats_explorer/chats_instance.html

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chats Protocols - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                {% if protocols %}
+                    <div class="d-flex justify-content-around my-2">
+                        {% for meta in protocols %}
+                            <a class="btn btn-light border-secondary" style="" href="#">
+                                <div class="card-body">
+                                    <h4 class="card-title">
+                                        {% if meta['icon'] %}
+                                            <i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
+                                        {% endif %}
+                                        <b>{{ meta['id'] }}</b>
+                                    </h4>
+                                </div>
+                            </a>
+                        {% endfor %}
+                    </div>
+
+                {% else %}
+                    <h4>No Protocol/Chats Imported</h4>
+                {% endif %}
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_cve").addClass("active");
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/chats_explorer/chats_protocols.html

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chats Protocols - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                {% if protocols %}
+                    <div class="d-flex justify-content-around my-2">
+                        {% for meta in protocols %}
+                            <a class="btn btn-light border-secondary" style="" href="#">
+                                <div class="card-body">
+                                    <h4 class="card-title">
+                                        {% if meta['icon'] %}
+                                            <i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
+                                        {% endif %}
+                                        <b>{{ meta['id'] }}</b>
+                                    </h4>
+                                </div>
+                            </a>
+                        {% endfor %}
+                    </div>
+
+                {% else %}
+                    <h4>No Protocol/Chats Imported</h4>
+                {% endif %}
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_cve").addClass("active");
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>