mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [chats] add messages threads
This commit is contained in:
parent
f766cbebda
commit
93ef541862
13 changed files with 329 additions and 44 deletions
|
@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from importer.feeders.Default import DefaultFeeder
|
from importer.feeders.Default import DefaultFeeder
|
||||||
from lib.objects.Chats import Chat
|
from lib.objects.Chats import Chat
|
||||||
from lib.objects import ChatSubChannels
|
from lib.objects import ChatSubChannels
|
||||||
|
from lib.objects import ChatThreads
|
||||||
from lib.objects import Images
|
from lib.objects import Images
|
||||||
from lib.objects import Messages
|
from lib.objects import Messages
|
||||||
from lib.objects import FilesNames
|
from lib.objects import FilesNames
|
||||||
|
@ -74,13 +75,13 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
return self.json_data['meta']['chat']['id']
|
return self.json_data['meta']['chat']['id']
|
||||||
|
|
||||||
def get_subchannel_id(self):
|
def get_subchannel_id(self):
|
||||||
pass
|
return self.json_data['meta']['chat'].get('subchannel', {}).get('id')
|
||||||
|
|
||||||
def get_subchannels(self):
|
def get_subchannels(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def get_thread_id(self):
|
def get_thread_id(self):
|
||||||
pass
|
return self.json_data['meta'].get('thread', {}).get('id')
|
||||||
|
|
||||||
def get_message_id(self):
|
def get_message_id(self):
|
||||||
return self.json_data['meta']['id']
|
return self.json_data['meta']['id']
|
||||||
|
@ -112,7 +113,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
return self.json_data['meta'].get('reply_to') # TODO change to reply ???
|
return self.json_data['meta'].get('reply_to') # TODO change to reply ???
|
||||||
|
|
||||||
def get_message_reply_id(self):
|
def get_message_reply_id(self):
|
||||||
return self.json_data['meta'].get('reply_to', None)
|
return self.json_data['meta'].get('reply_to', {}).get('message_id')
|
||||||
|
|
||||||
def get_message_content(self):
|
def get_message_content(self):
|
||||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||||
|
@ -125,6 +126,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
#### Create Object ID ####
|
#### Create Object ID ####
|
||||||
chat_id = self.get_chat_id()
|
chat_id = self.get_chat_id()
|
||||||
message_id = self.get_message_id()
|
message_id = self.get_message_id()
|
||||||
|
thread_id = self.get_thread_id()
|
||||||
# channel id
|
# channel id
|
||||||
# thread id
|
# thread id
|
||||||
|
|
||||||
|
@ -135,11 +137,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
self.obj = Images.Image(self.json_data['data-sha256'])
|
self.obj = Images.Image(self.json_data['data-sha256'])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
|
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, thread_id=thread_id)
|
||||||
self.obj = Messages.Message(obj_id)
|
self.obj = Messages.Message(obj_id)
|
||||||
return self.obj
|
return self.obj
|
||||||
|
|
||||||
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads
|
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
|
||||||
meta = self.json_data['meta']['chat'] # todo replace me by function
|
meta = self.json_data['meta']['chat'] # todo replace me by function
|
||||||
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
||||||
|
|
||||||
|
@ -170,6 +172,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
chat.add_children(obj_global_id=subchannel.get_global_id())
|
chat.add_children(obj_global_id=subchannel.get_global_id())
|
||||||
else:
|
else:
|
||||||
if obj.type == 'message':
|
if obj.type == 'message':
|
||||||
|
if self.get_thread_id():
|
||||||
|
self.process_thread(obj, chat, date, timestamp, reply_id=reply_id)
|
||||||
|
else:
|
||||||
chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
|
chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
|
||||||
|
|
||||||
|
|
||||||
|
@ -198,9 +203,32 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
subchannel.set_info(meta['info'])
|
subchannel.set_info(meta['info'])
|
||||||
|
|
||||||
if obj.type == 'message':
|
if obj.type == 'message':
|
||||||
|
if self.get_thread_id():
|
||||||
|
self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id)
|
||||||
|
else:
|
||||||
subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
|
subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
|
||||||
return subchannel
|
return subchannel
|
||||||
|
|
||||||
|
def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None):
|
||||||
|
meta = self.json_data['meta']['thread']
|
||||||
|
thread_id = self.get_thread_id()
|
||||||
|
p_chat_id = meta['parent'].get('chat')
|
||||||
|
p_subchannel_id = meta['parent'].get('subchannel')
|
||||||
|
p_message_id = meta['parent'].get('message')
|
||||||
|
|
||||||
|
if p_chat_id == self.get_chat_id() and p_subchannel_id == self.get_subchannel_id():
|
||||||
|
thread = ChatThreads.create(thread_id, self.get_chat_instance_uuid(), p_chat_id, p_subchannel_id, p_message_id, obj_chat)
|
||||||
|
thread.add(date, obj)
|
||||||
|
thread.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
|
||||||
|
# TODO OTHERS CORRELATIONS TO ADD
|
||||||
|
|
||||||
|
return thread
|
||||||
|
|
||||||
|
# TODO
|
||||||
|
# else:
|
||||||
|
# # ADD NEW MESSAGE REF (used by discord)
|
||||||
|
|
||||||
|
|
||||||
def process_sender(self, new_objs, obj, date, timestamp):
|
def process_sender(self, new_objs, obj, date, timestamp):
|
||||||
meta = self.json_data['meta']['sender']
|
meta = self.json_data['meta']['sender']
|
||||||
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
|
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
|
||||||
|
@ -298,6 +326,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
if media_name:
|
if media_name:
|
||||||
FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
|
FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
|
||||||
|
|
||||||
|
for reaction in self.get_reactions():
|
||||||
|
message.add_reaction(reaction['reaction'], int(reaction['count']))
|
||||||
|
|
||||||
for obj in objs: # TODO PERF avoid parsing metas multiple times
|
for obj in objs: # TODO PERF avoid parsing metas multiple times
|
||||||
|
|
||||||
# CHAT
|
# CHAT
|
||||||
|
|
|
@ -16,8 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon',
|
AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded',
|
||||||
'file-name', 'hhhash',
|
'domain', 'etag', 'favicon', 'file-name', 'hhhash',
|
||||||
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
|
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
|
||||||
|
|
||||||
def get_ail_uuid():
|
def get_ail_uuid():
|
||||||
|
|
|
@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib.objects import Chats
|
from lib.objects import Chats
|
||||||
from lib.objects import ChatSubChannels
|
from lib.objects import ChatSubChannels
|
||||||
|
from lib.objects import ChatThreads
|
||||||
from lib.objects import Messages
|
from lib.objects import Messages
|
||||||
from lib.objects import Usernames
|
from lib.objects import Usernames
|
||||||
|
|
||||||
|
@ -324,7 +325,7 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
|
||||||
def api_get_subchannel(chat_id, chat_instance_uuid):
|
def api_get_subchannel(chat_id, chat_instance_uuid):
|
||||||
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
|
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
|
||||||
if not subchannel.exists():
|
if not subchannel.exists():
|
||||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
return {"status": "error", "reason": "Unknown subchannel"}, 404
|
||||||
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
|
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
|
||||||
if meta['chat']:
|
if meta['chat']:
|
||||||
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
|
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
|
||||||
|
@ -333,6 +334,16 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
|
||||||
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
|
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
|
||||||
return meta, 200
|
return meta, 200
|
||||||
|
|
||||||
|
def api_get_thread(thread_id, thread_instance_uuid):
|
||||||
|
thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid)
|
||||||
|
if not thread.exists():
|
||||||
|
return {"status": "error", "reason": "Unknown thread"}, 404
|
||||||
|
meta = thread.get_meta({'chat', 'nb_messages'})
|
||||||
|
# if meta['chat']:
|
||||||
|
# meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
|
||||||
|
meta['messages'], meta['tags_messages'] = thread.get_messages()
|
||||||
|
return meta, 200
|
||||||
|
|
||||||
def api_get_message(message_id):
|
def api_get_message(message_id):
|
||||||
message = Messages.Message(message_id)
|
message = Messages.Message(message_id)
|
||||||
if not message.exists():
|
if not message.exists():
|
||||||
|
|
|
@ -41,7 +41,9 @@ config_loader = None
|
||||||
##################################
|
##################################
|
||||||
|
|
||||||
CORRELATION_TYPES_BY_OBJ = {
|
CORRELATION_TYPES_BY_OBJ = {
|
||||||
"chat": ["image", "user-account"], # message or direct correlation like cve, bitcoin, ... ???
|
"chat": ["chat-subchannel", "chat-thread", "image", "user-account"], # message or direct correlation like cve, bitcoin, ... ???
|
||||||
|
"chat-subchannel": ["chat", "chat-thread", "image", "message", "user-account"],
|
||||||
|
"chat-thread": ["chat", "chat-subchannel", "image", "message", "user-account"], # TODO user account
|
||||||
"cookie-name": ["domain"],
|
"cookie-name": ["domain"],
|
||||||
"cryptocurrency": ["domain", "item", "message"],
|
"cryptocurrency": ["domain", "item", "message"],
|
||||||
"cve": ["domain", "item", "message"],
|
"cve": ["domain", "item", "message"],
|
||||||
|
@ -53,11 +55,11 @@ CORRELATION_TYPES_BY_OBJ = {
|
||||||
"hhhash": ["domain"],
|
"hhhash": ["domain"],
|
||||||
"image": ["chat", "message", "user-account"],
|
"image": ["chat", "message", "user-account"],
|
||||||
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
|
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
|
||||||
"message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
|
"message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
|
||||||
"pgp": ["domain", "item", "message"],
|
"pgp": ["domain", "item", "message"],
|
||||||
"screenshot": ["domain", "item"],
|
"screenshot": ["domain", "item"],
|
||||||
"title": ["domain", "item"],
|
"title": ["domain", "item"],
|
||||||
"user-account": ["chat", "message"],
|
"user-account": ["chat", "chat-subchannel", "chat-thread", "message"],
|
||||||
"username": ["domain", "item", "message"], # TODO chat-user/account
|
"username": ["domain", "item", "message"], # TODO chat-user/account
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -149,7 +149,7 @@ class ChatSubChannel(AbstractChatObject):
|
||||||
|
|
||||||
class ChatSubChannels(AbstractChatObjects):
|
class ChatSubChannels(AbstractChatObjects):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__('chat-subchannels')
|
super().__init__('chat-subchannel')
|
||||||
|
|
||||||
# if __name__ == '__main__':
|
# if __name__ == '__main__':
|
||||||
# chat = Chat('test', 'telegram')
|
# chat = Chat('test', 'telegram')
|
||||||
|
|
|
@ -15,12 +15,8 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
from lib import ail_core
|
from lib import ail_core
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
|
||||||
from lib.data_retention_engine import update_obj_date
|
|
||||||
from lib.objects import ail_objects
|
|
||||||
from lib.timeline_engine import Timeline
|
|
||||||
|
|
||||||
from lib.correlations_engine import get_correlation_by_correl_type
|
|
||||||
|
|
||||||
config_loader = ConfigLoader()
|
config_loader = ConfigLoader()
|
||||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||||
|
@ -33,13 +29,13 @@ config_loader = None
|
||||||
################################################################################
|
################################################################################
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
class ChatThread(AbstractChatObject):
|
||||||
"""
|
"""
|
||||||
AIL Chat Object. (strings)
|
AIL Chat Object. (strings)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, id, subtype):
|
def __init__(self, id, subtype):
|
||||||
super(Chat, self).__init__('chat-thread', id, subtype)
|
super().__init__('chat-thread', id, subtype)
|
||||||
|
|
||||||
# def get_ail_2_ail_payload(self):
|
# def get_ail_2_ail_payload(self):
|
||||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
@ -69,7 +65,7 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
||||||
# style = 'fas'
|
# style = 'fas'
|
||||||
# icon = '\uf007'
|
# icon = '\uf007'
|
||||||
style = 'fas'
|
style = 'fas'
|
||||||
icon = '\uf086'
|
icon = '\uf7a4'
|
||||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
||||||
|
|
||||||
def get_meta(self, options=set()):
|
def get_meta(self, options=set()):
|
||||||
|
@ -77,27 +73,42 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
||||||
meta['id'] = self.id
|
meta['id'] = self.id
|
||||||
meta['subtype'] = self.subtype
|
meta['subtype'] = self.subtype
|
||||||
meta['tags'] = self.get_tags(r_list=True)
|
meta['tags'] = self.get_tags(r_list=True)
|
||||||
if 'username':
|
if 'nb_messages':
|
||||||
meta['username'] = self.get_username()
|
meta['nb_messages'] = self.get_nb_messages()
|
||||||
|
# created_at ???
|
||||||
return meta
|
return meta
|
||||||
|
|
||||||
def get_misp_object(self):
|
def get_misp_object(self):
|
||||||
return
|
return
|
||||||
|
|
||||||
############################################################################
|
def create(self, container_obj, message_id):
|
||||||
############################################################################
|
if message_id:
|
||||||
|
parent_message = container_obj.get_obj_by_message_id(message_id)
|
||||||
|
if parent_message: # TODO EXCEPTION IF DON'T EXISTS
|
||||||
|
self.set_parent(obj_global_id=parent_message)
|
||||||
|
_, _, parent_id = parent_message.split(':', 2)
|
||||||
|
self.add_correlation('message', '', parent_id)
|
||||||
|
else:
|
||||||
|
self.set_parent(obj_global_id=container_obj.get_global_id())
|
||||||
|
self.add_correlation(container_obj.get_type(), container_obj.get_subtype(r_str=True), container_obj.get_id())
|
||||||
|
|
||||||
# others optional metas, ... -> # TODO ALL meta in hset
|
def create(thread_id, chat_instance, chat_id, subchannel_id, message_id, container_obj):
|
||||||
|
if container_obj.get_type() == 'chat':
|
||||||
|
new_thread_id = f'{chat_id}/{thread_id}'
|
||||||
|
# sub-channel
|
||||||
|
else:
|
||||||
|
new_thread_id = f'{chat_id}/{subchannel_id}/{thread_id}'
|
||||||
|
|
||||||
#### Messages #### TODO set parents
|
thread = ChatThread(new_thread_id, chat_instance)
|
||||||
|
if not thread.exists():
|
||||||
|
thread.create(container_obj, message_id)
|
||||||
|
return thread
|
||||||
|
|
||||||
# def get_last_message_id(self):
|
class ChatThreads(AbstractChatObjects):
|
||||||
#
|
def __init__(self):
|
||||||
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
|
super().__init__('chat-thread')
|
||||||
|
|
||||||
|
# if __name__ == '__main__':
|
||||||
|
# chat = Chat('test', 'telegram')
|
||||||
if __name__ == '__main__':
|
# r = chat.get_messages()
|
||||||
chat = Chat('test', 'telegram')
|
# print(r)
|
||||||
r = chat.get_messages()
|
|
||||||
print(r)
|
|
||||||
|
|
|
@ -100,6 +100,13 @@ class Message(AbstractObject):
|
||||||
chat_id = self.get_basename().rsplit('_', 1)[0]
|
chat_id = self.get_basename().rsplit('_', 1)[0]
|
||||||
return chat_id
|
return chat_id
|
||||||
|
|
||||||
|
def get_thread(self):
|
||||||
|
for child in self.get_childrens():
|
||||||
|
obj_type, obj_subtype, obj_id = child.split(':', 2)
|
||||||
|
if obj_type == 'chat-thread':
|
||||||
|
nb_messages = r_object.zcard(f'messages:{obj_type}:{obj_subtype}:{obj_id}')
|
||||||
|
return {'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'nb': nb_messages}
|
||||||
|
|
||||||
# TODO get Instance ID
|
# TODO get Instance ID
|
||||||
# TODO get channel ID
|
# TODO get channel ID
|
||||||
# TODO get thread ID
|
# TODO get thread ID
|
||||||
|
@ -245,6 +252,10 @@ class Message(AbstractObject):
|
||||||
meta['user-account'] = {'id': 'UNKNOWN'}
|
meta['user-account'] = {'id': 'UNKNOWN'}
|
||||||
if 'chat' in options:
|
if 'chat' in options:
|
||||||
meta['chat'] = self.get_chat_id()
|
meta['chat'] = self.get_chat_id()
|
||||||
|
if 'thread' in options:
|
||||||
|
thread = self.get_thread()
|
||||||
|
if thread:
|
||||||
|
meta['thread'] = thread
|
||||||
if 'images' in options:
|
if 'images' in options:
|
||||||
meta['images'] = self.get_images()
|
meta['images'] = self.get_images()
|
||||||
if 'files-names' in options:
|
if 'files-names' in options:
|
||||||
|
@ -318,10 +329,10 @@ class Message(AbstractObject):
|
||||||
def delete(self):
|
def delete(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None):
|
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): # TODO CHECK COLLISIONS
|
||||||
timestamp = int(timestamp)
|
timestamp = int(timestamp)
|
||||||
if channel_id and thread_id:
|
if channel_id and thread_id:
|
||||||
return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ?????
|
return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}'
|
||||||
elif channel_id:
|
elif channel_id:
|
||||||
return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
|
return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
|
||||||
elif thread_id:
|
elif thread_id:
|
||||||
|
@ -329,6 +340,10 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
|
||||||
else:
|
else:
|
||||||
return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
|
return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
|
||||||
|
|
||||||
|
# thread id of message
|
||||||
|
# thread id of chat
|
||||||
|
# thread id of subchannel
|
||||||
|
|
||||||
# TODO Check if already exists
|
# TODO Check if already exists
|
||||||
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
|
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
|
||||||
def create(obj_id, content, translation=None, tags=[]):
|
def create(obj_id, content, translation=None, tags=[]):
|
||||||
|
|
|
@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
||||||
|
|
||||||
def get_message_meta(self, message, timestamp=None): # TODO handle file message
|
def get_message_meta(self, message, timestamp=None): # TODO handle file message
|
||||||
message = Messages.Message(message[9:])
|
message = Messages.Message(message[9:])
|
||||||
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp)
|
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'}, timestamp=timestamp)
|
||||||
return meta
|
return meta
|
||||||
|
|
||||||
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page
|
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page
|
||||||
|
@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
||||||
tags = {}
|
tags = {}
|
||||||
messages = {}
|
messages = {}
|
||||||
curr_date = None
|
curr_date = None
|
||||||
for message in self._get_messages(nb=50, page=1):
|
for message in self._get_messages(nb=2000, page=1):
|
||||||
timestamp = message[1]
|
timestamp = message[1]
|
||||||
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
|
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
|
||||||
if date_day != curr_date:
|
if date_day != curr_date:
|
||||||
|
|
|
@ -180,9 +180,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
||||||
self.add_correlation('domain', '', domain)
|
self.add_correlation('domain', '', domain)
|
||||||
|
|
||||||
# TODO:ADD objects + Stats
|
# TODO:ADD objects + Stats
|
||||||
def create(self, first_seen, last_seen):
|
# def create(self, first_seen, last_seen):
|
||||||
self.set_first_seen(first_seen)
|
# self.set_first_seen(first_seen)
|
||||||
self.set_last_seen(last_seen)
|
# self.set_last_seen(last_seen)
|
||||||
|
|
||||||
def _delete(self):
|
def _delete(self):
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -14,6 +14,8 @@ from lib import btc_ail
|
||||||
from lib import Tag
|
from lib import Tag
|
||||||
|
|
||||||
from lib.objects import Chats
|
from lib.objects import Chats
|
||||||
|
from lib.objects import ChatSubChannels
|
||||||
|
from lib.objects import ChatThreads
|
||||||
from lib.objects import CryptoCurrencies
|
from lib.objects import CryptoCurrencies
|
||||||
from lib.objects import CookiesNames
|
from lib.objects import CookiesNames
|
||||||
from lib.objects.Cves import Cve
|
from lib.objects.Cves import Cve
|
||||||
|
@ -62,6 +64,10 @@ def get_object(obj_type, subtype, obj_id):
|
||||||
return Decoded(obj_id)
|
return Decoded(obj_id)
|
||||||
elif obj_type == 'chat':
|
elif obj_type == 'chat':
|
||||||
return Chats.Chat(obj_id, subtype)
|
return Chats.Chat(obj_id, subtype)
|
||||||
|
elif obj_type == 'chat-subchannel':
|
||||||
|
return ChatSubChannels.ChatSubChannel(obj_id, subtype)
|
||||||
|
elif obj_type == 'chat-thread':
|
||||||
|
return ChatThreads.ChatThread(obj_id, subtype)
|
||||||
elif obj_type == 'cookie-name':
|
elif obj_type == 'cookie-name':
|
||||||
return CookiesNames.CookieName(obj_id)
|
return CookiesNames.CookieName(obj_id)
|
||||||
elif obj_type == 'cve':
|
elif obj_type == 'cve':
|
||||||
|
|
|
@ -112,6 +112,18 @@ def objects_subchannel_messages():
|
||||||
subchannel = subchannel[0]
|
subchannel = subchannel[0]
|
||||||
return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label)
|
return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
|
@chats_explorer.route("/chats/explorer/thread", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def objects_thread_messages():
|
||||||
|
thread_id = request.args.get('id')
|
||||||
|
instance_uuid = request.args.get('uuid')
|
||||||
|
thread = chats_viewer.api_get_thread(thread_id, instance_uuid)
|
||||||
|
if thread[1] != 200:
|
||||||
|
return create_json_response(thread[0], thread[1])
|
||||||
|
else:
|
||||||
|
meta = thread[0]
|
||||||
|
return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
@chats_explorer.route("/objects/message", methods=['GET'])
|
@chats_explorer.route("/objects/message", methods=['GET'])
|
||||||
@login_required
|
@login_required
|
||||||
|
|
191
var/www/templates/chats_explorer/ThreadMessages.html
Normal file
191
var/www/templates/chats_explorer/ThreadMessages.html
Normal file
|
@ -0,0 +1,191 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<title>Thread Messages - AIL</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
{# <link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
{# <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
|
||||||
|
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.chat-message-left,
|
||||||
|
.chat-message-right {
|
||||||
|
display: flex;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.chat-message-right {
|
||||||
|
flex-direction: row-reverse;
|
||||||
|
margin-left: auto
|
||||||
|
}
|
||||||
|
.divider:after,
|
||||||
|
.divider:before {
|
||||||
|
content: "";
|
||||||
|
flex: 1;
|
||||||
|
height: 2px;
|
||||||
|
background: #eee;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'sidebars/sidebar_objects.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||||
|
{{ meta["id"] }}
|
||||||
|
<ul class="list-group mb-2">
|
||||||
|
<li class="list-group-item py-0">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-10">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Parent</th>
|
||||||
|
<th>First seen</th>
|
||||||
|
<th>Last seen</th>
|
||||||
|
<th>Nb Messages</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
{{ meta['id'] }}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if meta['first_seen'] %}
|
||||||
|
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if meta['last_seen'] %}
|
||||||
|
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>{{ meta['nb_messages'] }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
{# <li class="list-group-item py-0">#}
|
||||||
|
{# <br>#}
|
||||||
|
{# <div class="mb-3">#}
|
||||||
|
{# Tags:#}
|
||||||
|
{# {% for tag in meta['tags'] %}#}
|
||||||
|
{# <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"#}
|
||||||
|
{# data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">#}
|
||||||
|
{# {{ tag }}#}
|
||||||
|
{# </button>#}
|
||||||
|
{# {% endfor %}#}
|
||||||
|
{# <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">#}
|
||||||
|
{# <i class="far fa-plus-square"></i>#}
|
||||||
|
{# </button>#}
|
||||||
|
{# </div>#}
|
||||||
|
{# </li>#}
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
{# {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}#}
|
||||||
|
{# {% include 'modals/investigations_register_obj.html' %}#}
|
||||||
|
{# {% endwith %}#}
|
||||||
|
{# <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
|
||||||
|
{# <i class="fas fa-microscope"></i> Investigations#}
|
||||||
|
{# </button>#}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% for tag in meta['tags_messages'] %}
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ meta['tags_messages'][tag] }}</span></span>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<div class="list-group d-inline-block">
|
||||||
|
{% for date in messages %}
|
||||||
|
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<span class="mt-3">
|
||||||
|
{% include 'objects/image/block_blur_img_slider.html' %}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<div class="position-relative">
|
||||||
|
<div class="chat-messages p-2">
|
||||||
|
|
||||||
|
{% for date in meta['messages'] %}
|
||||||
|
|
||||||
|
<div class="divider d-flex align-items-center mb-4">
|
||||||
|
<p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
|
||||||
|
<span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% for mess in meta['messages'][date] %}
|
||||||
|
|
||||||
|
{% with message=mess %}
|
||||||
|
{% include 'chats_explorer/block_message.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
|
<br>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var chart = {};
|
||||||
|
$(document).ready(function(){
|
||||||
|
$("#page-Decoded").addClass("active");
|
||||||
|
$("#nav_chat").addClass("active");
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
|
@ -74,6 +74,12 @@
|
||||||
{% for reaction in message['reactions'] %}
|
{% for reaction in message['reactions'] %}
|
||||||
<span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
|
<span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
{% if message['thread'] %}
|
||||||
|
<hr class="mb-1">
|
||||||
|
<div class="my-2 text-center">
|
||||||
|
<a href="{{ url_for('chats_explorer.objects_thread_messages')}}?uuid={{ message['thread']['subtype'] }}&id={{ message['thread']['id'] }}"><i class="far fa-comment"></i> {{ message['thread']['nb'] }} Messages</a>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
{% for tag in message['tags'] %}
|
{% for tag in message['tags'] %}
|
||||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
Loading…
Reference in a new issue