mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [translation] translate chats info, users info and subchannels names
This commit is contained in:
parent
6363a4f1cf
commit
896b411eaf
12 changed files with 95 additions and 23 deletions
|
@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from lib.ConfigLoader import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate')
|
||||
config_loader = None
|
||||
|
||||
|
@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True):
|
|||
content = content.replace(it, '')
|
||||
return content
|
||||
|
||||
#### AIL Objects ####
|
||||
|
||||
def get_obj_translation(obj_global_id, content, field='', source=None, target='en'):
|
||||
"""
|
||||
Returns translated content
|
||||
"""
|
||||
translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}')
|
||||
if translation:
|
||||
# DEBUG
|
||||
# print('cache')
|
||||
# r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0)
|
||||
return translation
|
||||
translation = LanguageTranslator().translate(content, source=source, target=target)
|
||||
if translation:
|
||||
r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation)
|
||||
r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300)
|
||||
return translation
|
||||
|
||||
## --AIL Objects-- ##
|
||||
|
||||
class LanguagesDetector:
|
||||
|
||||
|
|
|
@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()):
|
|||
options.add(option)
|
||||
return obj_chat.get_meta(options=options)
|
||||
|
||||
def get_subchannels_meta_from_global_id(subchannels):
|
||||
def get_subchannels_meta_from_global_id(subchannels, translation_target=None):
|
||||
meta = []
|
||||
for sub in subchannels:
|
||||
_, instance_uuid, sub_id = sub.split(':', 2)
|
||||
subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
|
||||
meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'}))
|
||||
meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target))
|
||||
return meta
|
||||
|
||||
def get_chat_meta_from_global_id(chat_global_id):
|
||||
|
@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
|
|||
chat = Chats.Chat(chat_id, chat_instance_uuid)
|
||||
if not chat.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'})
|
||||
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
|
||||
if meta['username']:
|
||||
meta['username'] = get_username_meta_from_global_id(meta['username'])
|
||||
if meta['subchannels']:
|
||||
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
|
||||
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target)
|
||||
else:
|
||||
if translation_target not in Language.LIST_LANGUAGES:
|
||||
if translation_target not in Language.get_translation_languages():
|
||||
translation_target = None
|
||||
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
|
||||
return meta, 200
|
||||
|
@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=
|
|||
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
|
||||
if not subchannel.exists():
|
||||
return {"status": "error", "reason": "Unknown subchannel"}, 404
|
||||
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'})
|
||||
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target)
|
||||
if meta['chat']:
|
||||
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
|
||||
if meta.get('threads'):
|
||||
|
@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None):
|
|||
meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target)
|
||||
return meta, 200
|
||||
|
||||
def api_get_user_account(user_id, instance_uuid):
|
||||
def api_get_user_account(user_id, instance_uuid, translation_target=None):
|
||||
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
|
||||
if not user_account.exists():
|
||||
return {"status": "error", "reason": "Unknown user-account"}, 404
|
||||
meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'})
|
||||
meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
|
||||
return meta, 200
|
||||
|
||||
# # # # # # # # # # LATER
|
||||
|
|
|
@ -76,7 +76,7 @@ class ChatSubChannel(AbstractChatObject):
|
|||
|
||||
# TODO TIME LAST MESSAGES
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
def get_meta(self, options=set(), translation_target=None):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
meta['name'] = self.get_name()
|
||||
|
@ -95,6 +95,8 @@ class ChatSubChannel(AbstractChatObject):
|
|||
meta['participants'] = self.get_participants()
|
||||
if 'nb_participants' in options:
|
||||
meta['nb_participants'] = self.get_nb_participants()
|
||||
if 'translation' in options and translation_target:
|
||||
meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target)
|
||||
return meta
|
||||
|
||||
def get_misp_object(self):
|
||||
|
|
|
@ -70,7 +70,7 @@ class Chat(AbstractChatObject):
|
|||
icon = '\uf086'
|
||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
def get_meta(self, options=set(), translation_target=None):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['name'] = self.get_name()
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
|
@ -79,6 +79,8 @@ class Chat(AbstractChatObject):
|
|||
meta['img'] = meta['icon']
|
||||
if 'info' in options:
|
||||
meta['info'] = self.get_info()
|
||||
if 'translation' in options and translation_target:
|
||||
meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
|
||||
if 'participants' in options:
|
||||
meta['participants'] = self.get_participants()
|
||||
if 'nb_participants' in options:
|
||||
|
|
|
@ -179,6 +179,7 @@ class Message(AbstractObject):
|
|||
"""
|
||||
Returns translated content
|
||||
"""
|
||||
|
||||
# return self._get_field('translated')
|
||||
global_id = self.get_global_id()
|
||||
translation = r_cache.get(f'translation:{target}:{global_id}')
|
||||
|
@ -289,7 +290,7 @@ class Message(AbstractObject):
|
|||
if 'reactions' in options:
|
||||
meta['reactions'] = self.get_reactions()
|
||||
if 'translation' in options and translation_target:
|
||||
meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target)
|
||||
meta['translation'] = self.translate(content=meta.get('content'), target=translation_target)
|
||||
|
||||
# meta['encoding'] = None
|
||||
return meta
|
||||
|
|
|
@ -5,6 +5,7 @@ import os
|
|||
import sys
|
||||
# import re
|
||||
|
||||
# from datetime import datetime
|
||||
from flask import url_for
|
||||
from pymisp import MISPObject
|
||||
|
||||
|
@ -88,6 +89,13 @@ class UserAccount(AbstractSubtypeObject):
|
|||
def set_info(self, info):
|
||||
return self._set_field('info', info)
|
||||
|
||||
# def get_created_at(self, date=False):
|
||||
# created_at = self._get_field('created_at')
|
||||
# if date and created_at:
|
||||
# created_at = datetime.fromtimestamp(float(created_at))
|
||||
# created_at = created_at.isoformat(' ')
|
||||
# return created_at
|
||||
|
||||
# TODO MESSAGES:
|
||||
# 1) ALL MESSAGES + NB
|
||||
# 2) ALL MESSAGES TIMESTAMP
|
||||
|
@ -122,7 +130,7 @@ class UserAccount(AbstractSubtypeObject):
|
|||
def get_messages_by_chat_obj(self, chat_obj):
|
||||
return self.get_correlation_iter_obj(chat_obj, 'message')
|
||||
|
||||
def get_meta(self, options=set()): # TODO Username timeline
|
||||
def get_meta(self, options=set(), translation_target=None): # TODO Username timeline
|
||||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['subtype'] = self.subtype
|
||||
|
@ -141,6 +149,10 @@ class UserAccount(AbstractSubtypeObject):
|
|||
meta['icon'] = self.get_icon()
|
||||
if 'info' in options:
|
||||
meta['info'] = self.get_info()
|
||||
if 'translation' in options and translation_target:
|
||||
meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
|
||||
# if 'created_at':
|
||||
# meta['created_at'] = self.get_created_at(date=True)
|
||||
if 'chats' in options:
|
||||
meta['chats'] = self.get_chats()
|
||||
if 'subchannels' in options:
|
||||
|
|
|
@ -25,6 +25,7 @@ from lib import Duplicate
|
|||
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
|
||||
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
|
||||
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
|
||||
from lib.Language import get_obj_translation
|
||||
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
|
||||
|
||||
logging.config.dictConfig(ail_logger.get_config(name='ail'))
|
||||
|
@ -301,6 +302,16 @@ class AbstractObject(ABC):
|
|||
|
||||
## -Relationship- ##
|
||||
|
||||
## Translation ##
|
||||
|
||||
def translate(self, content=None, field='', source=None, target='en'):
|
||||
global_id = self.get_global_id()
|
||||
if not content:
|
||||
content = self.get_content()
|
||||
return get_obj_translation(global_id, content, field=field, source=source, target=target)
|
||||
|
||||
## -Translation- ##
|
||||
|
||||
## Parent ##
|
||||
|
||||
def is_parent(self):
|
||||
|
|
|
@ -182,9 +182,14 @@ def objects_message():
|
|||
def objects_user_account():
|
||||
instance_uuid = request.args.get('subtype')
|
||||
user_id = request.args.get('id')
|
||||
user_account = chats_viewer.api_get_user_account(user_id, instance_uuid)
|
||||
target = request.args.get('target')
|
||||
if target == "Don't Translate":
|
||||
target = None
|
||||
user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
|
||||
if user_account[1] != 200:
|
||||
return create_json_response(user_account[0], user_account[1])
|
||||
else:
|
||||
user_account = user_account[0]
|
||||
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label)
|
||||
languages = Language.get_translation_languages()
|
||||
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
|
||||
translation_languages=languages, translation_target=target)
|
||||
|
|
|
@ -75,6 +75,9 @@
|
|||
<tr>
|
||||
<td>
|
||||
{{ subchannel['name'] }}
|
||||
{% if subchannel['translation_name'] %}
|
||||
<div class="text-secondary">{{ subchannel['translation_name'] }}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ subchannel["created_at"] }}</td>
|
||||
<td>
|
||||
|
|
|
@ -71,7 +71,7 @@
|
|||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>SubChannels</th>
|
||||
<th>Messages</th>
|
||||
<th><i class="fas fa-comment-dots"></i></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
|
|
|
@ -100,6 +100,10 @@
|
|||
{% if chat['info'] %}
|
||||
<li class="list-group-item py-0">
|
||||
<pre class="my-0">{{ chat['info'] }}</pre>
|
||||
{% if chat['translation_info'] %}
|
||||
<hr class="m-1">
|
||||
<pre class="my-0 text-secondary">{{ chat['translation_info'] }}</pre>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endif %}
|
||||
</li>
|
||||
|
@ -112,8 +116,12 @@
|
|||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ chat['tags_messages'][tag] }}</span></span>
|
||||
{% endfor %}
|
||||
|
||||
{% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
|
||||
{% include 'chats_explorer/block_translation.html' %}
|
||||
{% endwith %}
|
||||
|
||||
{% if chat['subchannels'] %}
|
||||
<h4>Sub-Channels:</h4>
|
||||
<h4 class="mt-2">Sub-Channels:</h4>
|
||||
<table id="tablesubchannels" class="table">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
|
@ -123,7 +131,7 @@
|
|||
<th>Created at</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Messages</th>
|
||||
<th><i class="fas fa-comment-dots"></i></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
|
@ -132,7 +140,12 @@
|
|||
<td>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
|
||||
</td>
|
||||
<td><b>{{ meta['name'] }}</b></td>
|
||||
<td>
|
||||
<b>{{ meta['name'] }}</b>
|
||||
{% if meta['translation_name'] %}
|
||||
<div class="text-secondary">{{ meta['translation_name'] }}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
|
||||
<td>{{ meta['created_at'] }}</td>
|
||||
<td>
|
||||
|
@ -161,9 +174,6 @@
|
|||
{% include 'objects/image/block_blur_img_slider.html' %}
|
||||
</span>
|
||||
|
||||
{% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
|
||||
{% include 'chats_explorer/block_translation.html' %}
|
||||
{% endwith %}
|
||||
{% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %}
|
||||
{% set date_from=chat['first_seen'] %}
|
||||
{% set date_to=chat['last_seen'] %}
|
||||
|
|
|
@ -46,7 +46,6 @@
|
|||
<tr>
|
||||
<th>username</th>
|
||||
<th>ID</th>
|
||||
<th>Created at</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Chats</th>
|
||||
|
@ -56,7 +55,6 @@
|
|||
<tr>
|
||||
<td>{{ meta['username']['id'] }}</td>
|
||||
<td>{{ meta['id'] }}</td>
|
||||
<td>{{ meta['created_at'] }}</td>
|
||||
<td>
|
||||
{% if meta['first_seen'] %}
|
||||
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
|
||||
|
@ -74,6 +72,10 @@
|
|||
{% if meta['info'] %}
|
||||
<li class="list-group-item py-0">
|
||||
<pre class="my-0">{{ meta['info'] }}</pre>
|
||||
{% if meta['translation_info'] %}
|
||||
<hr>
|
||||
<pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endif %}
|
||||
</li>
|
||||
|
@ -100,6 +102,10 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
|
||||
{% include 'chats_explorer/block_translation.html' %}
|
||||
{% endwith %}
|
||||
|
||||
|
||||
{# {% if meta['subchannels'] %}#}
|
||||
{# <h4>Sub-Channels:</h4>#}
|
||||
|
|
Loading…
Reference in a new issue