chg: [messages] refactor get_messages_meta + add basic message template

This commit is contained in:
terrtia 2023-11-08 10:31:51 +01:00
parent 4cc9608a3f
commit e7f060c23d
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
9 changed files with 82 additions and 132 deletions

View file

@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ConfigLoader import ConfigLoader
from lib.objects import Chats
from lib.objects import ChatSubChannels
from lib.objects import Messages
config_loader = ConfigLoader()
r_db = config_loader.get_db_conn("Kvrocks_DB")
@ -314,6 +315,12 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
return meta, 200
def api_get_message(message_id):
message = Messages.Message(message_id)
if not message.exists():
return {"status": "error", "reason": "Unknown uuid"}, 404
return message.get_meta({'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}), 200
# # # # # # # # # # LATER
# #
# ChatCategory #

View file

@ -155,43 +155,6 @@ class Chat(AbstractChatObject):
#
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
def _get_message_timestamp(self, obj_global_id):
return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
obj = ail_objects.get_obj_from_global_id(obj_global_id)
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
if mess_dict.get('parent') and parent:
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
if mess_dict.get('user-account'):
user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
mess_dict['user-account'] = {}
mess_dict['user-account']['type'] = user_account.get_type()
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
mess_dict['user-account']['id'] = user_account.get_id()
username = user_account.get_username()
if username:
username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
else:
mess_dict['user-account']['id'] = 'UNKNOWN'
if not mess_datetime:
obj_mess_id = self._get_message_timestamp(obj_global_id)
mess_datetime = datetime.fromtimestamp(obj_mess_id)
mess_dict['date'] = mess_datetime.isoformat(' ')
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
return mess_dict
# Zset with ID ??? id -> item id ??? multiple id == media + text
# id -> media id
# How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
# Handle media ???
# list of message id -> obj_id
# list of obj_id ->
# abuse parent children ???
# def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None):
# date = # TODO get date from object
# self.update_daterange(date)

View file

@ -18,6 +18,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ail_core import get_ail_uuid
from lib.objects.abstract_object import AbstractObject
from lib.ConfigLoader import ConfigLoader
from lib.objects import UsersAccount
from lib.data_retention_engine import update_obj_date, get_obj_date_first
# TODO Set all messages ???
@ -105,10 +106,14 @@ class Message(AbstractObject):
# TODO get channel ID
# TODO get thread ID
def get_user_account(self):
def get_user_account(self, meta=False):
user_account = self.get_correlation('user-account')
if user_account.get('user-account'):
return f'user-account:{user_account["user-account"].pop()}'
user_account = f'user-account:{user_account["user-account"].pop()}'
if meta:
_, user_account_subtype, user_account_id = user_account.split(':', 3)
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'})
return user_account
# Update value on import
# reply to -> parent ?
@ -176,26 +181,47 @@ class Message(AbstractObject):
# return r_object.hget(f'meta:item::{self.id}', 'url')
# options: set of optional meta fields
def get_meta(self, options=None):
def get_meta(self, options=None, timestamp=None):
"""
:type options: set
:type timestamp: float
"""
if options is None:
options = set()
meta = self.get_default_meta(tags=True)
meta['date'] = self.get_date()
# timestamp
if not timestamp:
timestamp = self.get_timestamp()
else:
timestamp = float(timestamp)
timestamp = datetime.fromtimestamp(float(timestamp))
meta['date'] = timestamp.strftime('%Y%m%d')
meta['hour'] = timestamp.strftime('%H:%M:%S')
meta['full_date'] = timestamp.isoformat(' ')
meta['source'] = self.get_source()
# optional meta fields
if 'content' in options:
meta['content'] = self.get_content()
if 'parent' in options:
meta['parent'] = self.get_parent()
if meta['parent'] and 'parent_meta' in options:
options.remove('parent')
parent_type, _, parent_id = meta['parent'].split(':', 3)
if parent_type == 'message':
message = Message(parent_id)
meta['reply_to'] = message.get_meta(options=options)
if 'investigations' in options:
meta['investigations'] = self.get_investigations()
if 'link' in options:
meta['link'] = self.get_link(flask_context=True)
if 'icon' in options:
meta['icon'] = self.get_svg_icon()
if 'user-account' in options:
meta['user-account'] = self.get_user_account()
meta['user-account'] = self.get_user_account(meta=True)
if not meta['user-account']:
meta['user-account'] = {'id': 'UNKNOWN'}
# meta['encoding'] = None
return meta

View file

@ -16,6 +16,8 @@ from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.timeline_engine import Timeline
from lib.objects import Usernames
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@ -97,9 +99,12 @@ class UserAccount(AbstractSubtypeObject):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True)
meta['tags'] = self.get_tags(r_list=True) # TODO add in options ????
if 'username' in options:
meta['username'] = self.get_username()
if meta['username'] and 'username_meta' in options:
_, username_account_subtype, username_account_id = meta['username'].split(':', 3)
meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta()
if 'usernames' in options:
meta['usernames'] = self.get_usernames()
return meta

View file

@ -21,11 +21,9 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject
from lib.ail_core import get_object_all_subtypes, zscan_iter ################
from lib.ConfigLoader import ConfigLoader
from lib.objects import Messages
from lib.objects.UsersAccount import UserAccount
from lib.objects.Usernames import Username
from lib.data_retention_engine import update_obj_date
from packages import Date
# from lib.data_retention_engine import update_obj_date
# LOAD CONFIG
config_loader = ConfigLoader()
@ -143,33 +141,10 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def get_last_message(self):
return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message
obj = Messages.Message(message[9:])
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
# print(mess_dict)
if mess_dict.get('parent') and parent:
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
if mess_dict.get('user-account'):
_, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3)
user_account = UserAccount(user_account_id, user_account_subtype)
mess_dict['user-account'] = {}
mess_dict['user-account']['type'] = user_account.get_type()
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
mess_dict['user-account']['id'] = user_account.get_id()
username = user_account.get_username()
if username:
_, username_account_subtype, username_account_id = username.split(':', 3)
username = Username(username_account_id, username_account_subtype).get_default_meta(link=False)
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
else:
mess_dict['user-account'] = {'id': 'UNKNOWN'}
if not mess_datetime:
obj_mess_id = obj.get_timestamp()
mess_datetime = datetime.fromtimestamp(float(obj_mess_id))
mess_dict['date'] = mess_datetime.isoformat(' ')
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
return mess_dict
def get_message_meta(self, message, timestamp=None): # TODO handle file message
message = Messages.Message(message[9:])
meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
return meta
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ????
# TODO return message meta
@ -177,12 +152,12 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
messages = {}
curr_date = None
for message in self._get_messages():
date = datetime.fromtimestamp(message[1])
date_day = date.strftime('%Y/%m/%d')
timestamp = message[1]
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
if date_day != curr_date:
messages[date_day] = []
curr_date = date_day
mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object
mess_dict = self.get_message_meta(message[0], timestamp=timestamp)
messages[date_day].append(mess_dict)
if mess_dict.get('tags'):
@ -257,6 +232,3 @@ class AbstractChatObjects(ABC):
def search(self):
pass

View file

@ -108,26 +108,15 @@ def objects_subchannel_messages():
subchannel = subchannel[0]
return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label)
#############################################################################################
#############################################################################################
#############################################################################################
@chats_explorer.route("/objects/chat/messages", methods=['GET'])
@chats_explorer.route("/objects/message", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_chat():
chat = request.args.get('id')
subtype = request.args.get('subtype')
chat = Chats.Chat(chat, subtype)
if chat.exists():
messages, mess_tags = chat.get_messages()
print(messages)
print(chat.get_subchannels())
meta = chat.get_meta({'icon', 'username'})
if meta.get('username'):
meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta()
print(meta)
return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label)
message_id = request.args.get('id')
message = chats_viewer.api_get_message(message_id)
if message[1] != 200:
return create_json_response(message[0], message[1])
else:
return abort(404)
message = message[0]
return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label)

View file

@ -2,7 +2,7 @@
<html>
<head>
<title>Chat Messages - AIL</title>
<title>Chat Message - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
@ -112,63 +112,51 @@
</div>
</div>
{% for tag in mess_tags %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
{% endfor %}
<div>
<div class="list-group d-inline-block">
{% for date in messages %}
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
{% endfor %}
</div>
</div>
<div class="position-relative">
<div class="chat-messages p-4">
<div class="chat-messages p-2">
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
<span class="badge badge-secondary mb-2">{{ meta['date'] }}</span>
<div class="chat-message-left pb-1">
<div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ meta['hour'] }}</div>
</div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['user-account']['username'] %}
{{ mess['user-account']['username']['id'] }}
{% if meta['user-account']['username'] %}
{{ meta['user-account']['username']['id'] }}
{% else %}
{{ mess['user-account']['id'] }}
{{ meta['user-account']['id'] }}
{% endif %}
</div>
{% if mess['reply_to'] %}
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
{% if meta['reply_to'] %}
<div class="flex-shrink-1 bg-white border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['reply_to']['user-account']['username'] %}
{{ mess['reply_to']['user-account']['username']['id'] }}
{% if meta['reply_to']['user-account']['username'] %}
{{ meta['reply_to']['user-account']['username']['id'] }}
{% else %}
{{ mess['reply_to']['user-account']['id'] }}
{{ meta['reply_to']['user-account']['id'] }}
{% endif %}
</div>
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
{% for tag in mess['reply_to']['tags'] %}
<pre class="my-0">{{ meta['reply_to']['content'] }}</pre>
{% for tag in meta['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
<div class="text-muted small text-nowrap">{{ meta['reply_to']['full_date'] }}</div>
{# <div class="">#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
{# </div>#}
</div>
{% endif %}
<pre class="my-0">{{ mess['content'] }}</pre>
{% for tag in mess['tags'] %}
<pre class="my-0">{{ meta['content'] }}</pre>
{% for tag in meta['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="">
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}"><i class="fas fa-project-diagram"></i></a>
</div>
</div>
</div>

View file

@ -181,7 +181,7 @@
{% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
<div class="text-muted small text-nowrap">{{ mess['reply_to']['full_date'] }}</div>
{# <div class="">#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}

View file

@ -183,7 +183,7 @@
{% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
<div class="text-muted small text-nowrap">{{ mess['reply_to']['full_date'] }}</div>
{# <div class="">#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}