chg: [messages] refactor get_messages_meta + add basic message template

bin/lib/chats_viewer.py

@@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib.ConfigLoader import ConfigLoader
 from lib.objects import Chats
 from lib.objects import ChatSubChannels
+from lib.objects import Messages
 
 config_loader = ConfigLoader()
 r_db = config_loader.get_db_conn("Kvrocks_DB")
@@ -314,6 +315,12 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
     meta['messages'], meta['tags_messages'] = subchannel.get_messages()
     return meta, 200
 
+def api_get_message(message_id):
+    message = Messages.Message(message_id)
+    if not message.exists():
+        return {"status": "error", "reason": "Unknown uuid"}, 404
+    return message.get_meta({'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}), 200
+
 # # # # # # # # # # LATER
 #                 #
 #   ChatCategory  #

bin/lib/objects/Chats.py

@@ -155,43 +155,6 @@ class Chat(AbstractChatObject):
     #
     #     return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
 
-    def _get_message_timestamp(self, obj_global_id):
-        return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
-
-    def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
-        obj = ail_objects.get_obj_from_global_id(obj_global_id)
-        mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
-        if mess_dict.get('parent') and parent:
-            mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
-        if mess_dict.get('user-account'):
-            user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
-            mess_dict['user-account'] = {}
-            mess_dict['user-account']['type'] = user_account.get_type()
-            mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
-            mess_dict['user-account']['id'] = user_account.get_id()
-            username = user_account.get_username()
-            if username:
-                username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
-            mess_dict['user-account']['username'] = username  # TODO get username at the given timestamp ???
-        else:
-            mess_dict['user-account']['id'] = 'UNKNOWN'
-
-        if not mess_datetime:
-            obj_mess_id = self._get_message_timestamp(obj_global_id)
-            mess_datetime = datetime.fromtimestamp(obj_mess_id)
-        mess_dict['date'] = mess_datetime.isoformat(' ')
-        mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
-        return mess_dict
-
-        # Zset with ID ???  id -> item id ??? multiple id == media + text
-        #                   id -> media id
-        # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
-        # Handle media ???
-
-        # list of message id -> obj_id
-        # list of obj_id ->
-        # abuse parent children ???
-
     # def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None):
     #     date = # TODO get date from object
     #     self.update_daterange(date)

bin/lib/objects/Messages.py

@@ -18,6 +18,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib.ail_core import get_ail_uuid
 from lib.objects.abstract_object import AbstractObject
 from lib.ConfigLoader import ConfigLoader
+from lib.objects import UsersAccount
 from lib.data_retention_engine import update_obj_date, get_obj_date_first
 # TODO Set all messages ???
 
@@ -105,10 +106,14 @@ class Message(AbstractObject):
     # TODO get channel ID
     # TODO get thread  ID
 
-    def get_user_account(self):
+    def get_user_account(self, meta=False):
         user_account = self.get_correlation('user-account')
         if user_account.get('user-account'):
-            return f'user-account:{user_account["user-account"].pop()}'
+            user_account = f'user-account:{user_account["user-account"].pop()}'
+            if meta:
+                _, user_account_subtype, user_account_id = user_account.split(':', 3)
+                user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'})
+        return  user_account
 
     # Update value on import
     # reply to -> parent ?
@@ -176,26 +181,47 @@ class Message(AbstractObject):
     #     return r_object.hget(f'meta:item::{self.id}', 'url')
 
     # options: set of optional meta fields
-    def get_meta(self, options=None):
+    def get_meta(self, options=None, timestamp=None):
         """
         :type options: set
+        :type timestamp: float
         """
         if options is None:
             options = set()
         meta = self.get_default_meta(tags=True)
-        meta['date'] = self.get_date()
+
+        # timestamp
+        if not timestamp:
+            timestamp = self.get_timestamp()
+        else:
+            timestamp = float(timestamp)
+        timestamp = datetime.fromtimestamp(float(timestamp))
+        meta['date'] = timestamp.strftime('%Y%m%d')
+        meta['hour'] = timestamp.strftime('%H:%M:%S')
+        meta['full_date'] = timestamp.isoformat(' ')
+
         meta['source'] = self.get_source()
         # optional meta fields
         if 'content' in options:
             meta['content'] = self.get_content()
         if 'parent' in options:
             meta['parent'] = self.get_parent()
+            if meta['parent'] and 'parent_meta' in options:
+                options.remove('parent')
+                parent_type, _, parent_id = meta['parent'].split(':', 3)
+                if parent_type == 'message':
+                    message = Message(parent_id)
+                    meta['reply_to'] = message.get_meta(options=options)
         if 'investigations' in options:
             meta['investigations'] = self.get_investigations()
         if 'link' in options:
             meta['link'] = self.get_link(flask_context=True)
+        if 'icon' in options:
+            meta['icon'] = self.get_svg_icon()
         if 'user-account' in options:
-            meta['user-account'] = self.get_user_account()
+            meta['user-account'] = self.get_user_account(meta=True)
+            if not meta['user-account']:
+                meta['user-account'] = {'id': 'UNKNOWN'}
 
         # meta['encoding'] = None
         return meta

bin/lib/objects/UsersAccount.py

@@ -16,6 +16,8 @@ from lib import ail_core
 from lib.ConfigLoader import ConfigLoader
 from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 from lib.timeline_engine import Timeline
+from lib.objects import Usernames
+
 
 config_loader = ConfigLoader()
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@@ -97,9 +99,12 @@ class UserAccount(AbstractSubtypeObject):
         meta = self._get_meta(options=options)
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags(r_list=True)
+        meta['tags'] = self.get_tags(r_list=True)  # TODO add in options ????
         if 'username' in options:
             meta['username'] = self.get_username()
+            if meta['username'] and 'username_meta' in options:
+                _, username_account_subtype, username_account_id = meta['username'].split(':', 3)
+                meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta()
         if 'usernames' in options:
             meta['usernames'] = self.get_usernames()
         return meta

bin/lib/objects/abstract_chat_object.py

@@ -21,11 +21,9 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject
 from lib.ail_core import get_object_all_subtypes, zscan_iter ################
 from lib.ConfigLoader import ConfigLoader
 from lib.objects import Messages
-from lib.objects.UsersAccount import UserAccount
-from lib.objects.Usernames import Username
-from lib.data_retention_engine import update_obj_date
 
-from packages import Date
+# from lib.data_retention_engine import update_obj_date
+
 
 # LOAD CONFIG
 config_loader = ConfigLoader()
@@ -143,33 +141,10 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
     def get_last_message(self):
         return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
 
-    def get_message_meta(self, message, parent=True, mess_datetime=None):  # TODO handle file message
+    def get_message_meta(self, message, timestamp=None):  # TODO handle file message
-        obj = Messages.Message(message[9:])
+        message = Messages.Message(message[9:])
-        mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
+        meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
-        # print(mess_dict)
+        return meta
-        if mess_dict.get('parent') and parent:
-            mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
-        if mess_dict.get('user-account'):
-            _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3)
-            user_account = UserAccount(user_account_id, user_account_subtype)
-            mess_dict['user-account'] = {}
-            mess_dict['user-account']['type'] = user_account.get_type()
-            mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
-            mess_dict['user-account']['id'] = user_account.get_id()
-            username = user_account.get_username()
-            if username:
-                _, username_account_subtype, username_account_id = username.split(':', 3)
-                username = Username(username_account_id, username_account_subtype).get_default_meta(link=False)
-            mess_dict['user-account']['username'] = username  # TODO get username at the given timestamp ???
-        else:
-            mess_dict['user-account'] = {'id': 'UNKNOWN'}
-
-        if not mess_datetime:
-            obj_mess_id = obj.get_timestamp()
-            mess_datetime = datetime.fromtimestamp(float(obj_mess_id))
-        mess_dict['date'] = mess_datetime.isoformat(' ')
-        mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
-        return mess_dict
 
     def get_messages(self, start=0, page=1, nb=500, unread=False):  # threads ????
         # TODO return message meta
@@ -177,12 +152,12 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
         messages = {}
         curr_date = None
         for message in self._get_messages():
-            date = datetime.fromtimestamp(message[1])
+            timestamp = message[1]
-            date_day = date.strftime('%Y/%m/%d')
+            date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
             if date_day != curr_date:
                 messages[date_day] = []
                 curr_date = date_day
-            mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)  # TODO use object
+            mess_dict = self.get_message_meta(message[0], timestamp=timestamp)
             messages[date_day].append(mess_dict)
 
             if mess_dict.get('tags'):
@@ -257,6 +232,3 @@ class AbstractChatObjects(ABC):
 
     def search(self):
         pass
-
-
-

var/www/blueprints/chats_explorer.py

@@ -108,26 +108,15 @@ def objects_subchannel_messages():
         subchannel = subchannel[0]
         return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label)
 
-#############################################################################################
-#############################################################################################
-#############################################################################################
 
-
+@chats_explorer.route("/objects/message", methods=['GET'])
-@chats_explorer.route("/objects/chat/messages", methods=['GET'])
 @login_required
 @login_read_only
 def objects_dashboard_chat():
-    chat = request.args.get('id')
+    message_id = request.args.get('id')
-    subtype = request.args.get('subtype')
+    message = chats_viewer.api_get_message(message_id)
-    chat = Chats.Chat(chat, subtype)
+    if message[1] != 200:
-    if chat.exists():
+        return create_json_response(message[0], message[1])
-        messages, mess_tags = chat.get_messages()
-        print(messages)
-        print(chat.get_subchannels())
-        meta = chat.get_meta({'icon', 'username'})
-        if meta.get('username'):
-            meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta()
-        print(meta)
-        return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label)
     else:
-        return abort(404)
+        message = message[0]
+        return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label)

var/www/templates/chats_explorer/ChatMessage.html

@@ -2,7 +2,7 @@
 <html>
 
 <head>
-  <title>Chat Messages - AIL</title>
+  <title>Chat Message - AIL</title>
 	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
@@ -112,63 +112,51 @@
                     </div>
                 </div>
 
-                {% for tag in mess_tags %}
-                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
-                {%  endfor %}
-
-                <div>
-                    <div class="list-group d-inline-block">
-                        {% for date in messages %}
-                            <a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
-                        {% endfor %}
-                    </div>
-                </div>
 
                 <div class="position-relative">
-                    <div class="chat-messages p-4">
+                    <div class="chat-messages p-2">
 
-                        <h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
+                        <span class="badge badge-secondary mb-2">{{ meta['date'] }}</span>
 
                         <div class="chat-message-left pb-1">
                             <div>
-                                <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
+                                <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['user-account']['id'] }}" width="40" height="40">
-                                <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
+                                <div class="text-muted small text-nowrap mt-2">{{ meta['hour'] }}</div>
                             </div>
                             <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
                                 <div class="font-weight-bold mb-1">
-                                    {% if mess['user-account']['username'] %}
+                                    {% if meta['user-account']['username'] %}
-                                        {{ mess['user-account']['username']['id'] }}
+                                        {{ meta['user-account']['username']['id'] }}
                                     {% else %}
-                                        {{ mess['user-account']['id'] }}
+                                        {{ meta['user-account']['id'] }}
                                     {% endif %}
                                 </div>
-                                {% if mess['reply_to'] %}
+                                {% if meta['reply_to'] %}
-                                    <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
+                                    <div class="flex-shrink-1 bg-white border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
                                         <div class="font-weight-bold mb-1">
-                                            {% if mess['reply_to']['user-account']['username'] %}
+                                            {% if meta['reply_to']['user-account']['username'] %}
-                                                {{ mess['reply_to']['user-account']['username']['id'] }}
+                                                {{ meta['reply_to']['user-account']['username']['id'] }}
                                             {% else %}
-                                                {{ mess['reply_to']['user-account']['id'] }}
+                                                {{ meta['reply_to']['user-account']['id'] }}
                                             {% endif %}
                                         </div>
-                                        <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
+                                        <pre class="my-0">{{ meta['reply_to']['content'] }}</pre>
-                                        {% for tag in mess['reply_to']['tags'] %}
+                                        {% for tag in meta['reply_to']['tags'] %}
                                             <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
                                         {%  endfor %}
-                                        <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+                                        <div class="text-muted small text-nowrap">{{ meta['reply_to']['full_date'] }}</div>
 {#                                                <div class="">#}
 {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
 {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
 {#                                                </div>#}
                                     </div>
                                 {% endif %}
-                                <pre class="my-0">{{ mess['content'] }}</pre>
+                                <pre class="my-0">{{ meta['content'] }}</pre>
-                                {% for tag in mess['tags'] %}
+                                {% for tag in meta['tags'] %}
                                     <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
                                 {%  endfor %}
                                 <div class="">
-                                    <a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
+                                    <a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}"><i class="fas fa-project-diagram"></i></a>
-                                    <a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
                                 </div>
                             </div>
                         </div>

var/www/templates/chats_explorer/SubChannelMessages.html

@@ -181,7 +181,7 @@
                                                 {% for tag in mess['reply_to']['tags'] %}
                                                     <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
                                                 {%  endfor %}
-                                                <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+                                                <div class="text-muted small text-nowrap">{{ mess['reply_to']['full_date'] }}</div>
 {#                                                <div class="">#}
 {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
 {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}

var/www/templates/chats_explorer/chat_viewer.html

@@ -183,7 +183,7 @@
                                                     {% for tag in mess['reply_to']['tags'] %}
                                                         <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
                                                     {%  endfor %}
-                                                    <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+                                                    <div class="text-muted small text-nowrap">{{ mess['reply_to']['full_date'] }}</div>
     {#                                                <div class="">#}
     {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
     {#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}