AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [chats] add message file-name object + str emoticon reactions

Browse source
This commit is contained in:
terrtia 2023-11-27 16:25:09 +01:00
parent 235a913865
commit f766cbebda
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
9 changed files with 175 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
bin/importer/feeders/abstract_chats_feeder.py
View file

@ -22,6 +22,8 @@ from lib.objects.Chats import Chat
from lib.objects import ChatSubChannels
from lib.objects import Images
from lib.objects import Messages
from lib.objects import FilesNames
# from lib.objects import Files
from lib.objects import UsersAccount
from lib.objects.Usernames import Username
from lib import chats_viewer
@ -83,6 +85,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
def get_message_id(self):
return self.json_data['meta']['id']
def get_media_name(self):
return self.json_data['meta'].get('media', {}).get('name')
def get_reactions(self):
return self.json_data['meta'].get('reactions', [])
def get_message_timestamp(self):
return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
# if self.json_data['meta'].get('date'):
@ -223,6 +231,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
user_account.set_icon(img.get_global_id())
new_objs.add(img)
if meta.get('info'):
user_account.set_info(meta['info'])
return user_account
# Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
@ -251,11 +262,22 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
print(self.obj.type)
# TODO FILES + FILES REF
# get object by meta object type
if self.obj.type == 'message':
# Content
obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation
# FILENAME
media_name = self.get_media_name()
if media_name:
print(media_name)
FilesNames.FilesNames().create(media_name, date, obj)
for reaction in self.get_reactions():
obj.add_reaction(reaction['reaction'], int(reaction['count']))
else:
chat_id = self.get_chat_id()
message_id = self.get_message_id()
@ -271,6 +293,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
obj.add(date, message)
obj.set_parent(obj_global_id=message.get_global_id())
# FILENAME
media_name = self.get_media_name()
if media_name:
FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
for obj in objs: # TODO PERF avoid parsing metas multiple times
# CHAT

3
bin/lib/ail_core.py
View file

@ -16,7 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash',
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon',
'file-name', 'hhhash',
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
def get_ail_uuid():

3
bin/lib/correlations_engine.py
View file

@ -49,10 +49,11 @@ CORRELATION_TYPES_BY_OBJ = {
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
"etag": ["domain"],
"favicon": ["domain", "item"], # TODO Decoded
"file-name": ["chat", "message"],
"hhhash": ["domain"],
"image": ["chat", "message", "user-account"],
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
"message": ["cve", "cryptocurrency", "decoded", "image", "pgp", "user-account"], # chat ??
"message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
"pgp": ["domain", "item", "message"],
"screenshot": ["domain", "item"],
"title": ["domain", "item"],

101
bin/lib/objects/FilesNames.py Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
from flask import url_for
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
config_loader = ConfigLoader()
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
class FileName(AbstractDaterangeObject):
"""
AIL FileName Object. (strings)
"""
# ID = SHA256
def __init__(self, name):
super().__init__('file-name', name)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', type=self.type, id=self.id)
else:
url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
return url
def get_svg_icon(self):
return {'style': 'far', 'icon': '\uf249', 'color': '#36F5D5', 'radius': 5}
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('file')
# obj_attrs.append(obj.add_attribute('sha256', value=self.id))
# obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content()))
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
return obj
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['tags'] = self.get_tags(r_list=True)
if 'tags_safe' in options:
meta['tags_safe'] = self.is_tags_safe(meta['tags'])
return meta
def create(self): # create ALL SET ??????
pass
def add_reference(self, date, src_ail_object, file_obj=None):
self.add(date, src_ail_object)
if file_obj:
self.add_correlation(file_obj.type, file_obj.get_subtype(r_str=True), file_obj.get_id())
# TODO USE ZSET FOR ALL OBJS IDS ??????
class FilesNames(AbstractDaterangeObjects):
"""
CookieName Objects
"""
def __init__(self):
super().__init__('file-name', FileName)
def sanitize_id_to_search(self, name_to_search):
return name_to_search
# TODO sanitize file name
def create(self, name, date, src_ail_object, file_obj=None, limit=500, force=False):
if 0 < len(name) <= limit or force or limit < 0:
file_name = self.obj_class(name)
# if not file_name.exists():
# file_name.create()
file_name.add_reference(date, src_ail_object, file_obj=file_obj)
return file_name
# if __name__ == '__main__':
# name_to_search = '29ba'
# print(search_screenshots_by_name(name_to_search))

23
bin/lib/objects/Messages.py
View file

@ -118,9 +118,24 @@ class Message(AbstractObject):
user_account = f'user-account:{user_account["user-account"].pop()}'
if meta:
_, user_account_subtype, user_account_id = user_account.split(':', 3)
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'})
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
return user_account
def get_files_names(self):
names = []
filenames = self.get_correlation('file-name').get('file-name')
if filenames:
for name in filenames:
names.append(name[1:])
return names
def get_reactions(self):
return r_object.hgetall(f'meta:reactions:{self.type}::{self.id}')
# TODO sanitize reactions
def add_reaction(self, reactions, nb_reaction):
r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
# Update value on import
# reply to -> parent ?
# reply/comment - > children ?
@ -232,6 +247,10 @@ class Message(AbstractObject):
meta['chat'] = self.get_chat_id()
if 'images' in options:
meta['images'] = self.get_images()
if 'files-names' in options:
meta['files-names'] = self.get_files_names()
if 'reactions' in options:
meta['reactions'] = self.get_reactions()
# meta['encoding'] = None
return meta
@ -314,7 +333,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
def create(obj_id, content, translation=None, tags=[]):
message = Message(obj_id)
if not message.exists():
# if not message.exists():
message.create(content, translation=translation, tags=tags)
return message

6
bin/lib/objects/UsersAccount.py
View file

@ -91,6 +91,12 @@ class UserAccount(AbstractSubtypeObject):
def set_icon(self, icon):
self._set_field('icon', icon)
def get_info(self):
return self._get_field('info')
def set_info(self, info):
return self._set_field('info', info)
def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username')

4
bin/lib/objects/abstract_chat_object.py
View file

@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def get_message_meta(self, message, timestamp=None): # TODO handle file message
message = Messages.Message(message[9:])
meta = message.get_meta(options={'content', 'images', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp)
return meta
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page
@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
tags = {}
messages = {}
curr_date = None
for message in self._get_messages(nb=30, page=1):
for message in self._get_messages(nb=50, page=1):
timestamp = message[1]
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
if date_day != curr_date:

3
bin/lib/objects/ail_objects.py
View file

@ -21,6 +21,7 @@ from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decod
from lib.objects.Domains import Domain
from lib.objects import Etags
from lib.objects.Favicons import Favicon
from lib.objects import FilesNames
from lib.objects import HHHashs
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects import Images
@ -69,6 +70,8 @@ def get_object(obj_type, subtype, obj_id):
return Etags.Etag(obj_id)
elif obj_type == 'favicon':
return Favicon(obj_id)
elif obj_type == 'file-name':
return FilesNames.FileName(obj_id)
elif obj_type == 'hhhash':
return HHHashs.HHHash(obj_id)
elif obj_type == 'image':

10
var/www/templates/chats_explorer/block_message.html
View file

@ -63,7 +63,17 @@
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
{% endfor %}
{% endif %}
{% if message['files-names'] %}
{% for file_name in message['files-names'] %}
<div class="flex-shrink-1 bg-white border-primary text-secondary rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<i class="far fa-file fa-3x"></i> {{ file_name }}
</div>
{% endfor %}
{% endif %}
<pre class="my-0">{{ message['content'] }}</pre>
{% for reaction in message['reactions'] %}
<span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
{% endfor %}
{% for tag in message['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}