From b9c37167ad3e84a5b8fc53b9418f12a208f1c073 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 25 Mar 2024 14:13:16 +0100 Subject: [PATCH] chg: [language messages] add nb languages stats by chat/subchannel objects --- bin/lib/Language.py | 43 +++++++++----- bin/lib/chats_viewer.py | 1 - bin/lib/objects/Messages.py | 58 +++++++++++-------- bin/lib/objects/abstract_object.py | 21 ++++--- var/www/blueprints/chats_explorer.py | 10 +++- .../chats_explorer/block_message.html | 2 +- 6 files changed, 87 insertions(+), 48 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 5d3d29c9..f6c9ebfb 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -324,24 +324,32 @@ def get_objs_languages(obj_type, obj_subtype=''): def get_obj_languages(obj_type, obj_subtype, obj_id): return r_lang.smembers(f'obj:lang:{obj_type}:{obj_subtype}:{obj_id}') +def get_obj_language_stats(obj_type, obj_subtype, obj_id): + return r_lang.zrange(f'obj:langs:stat:{obj_type}:{obj_subtype}:{obj_id}', 0, -1, withscores=True) + # TODO ADD language to CHAT GLOBAL SET -def add_obj_language(language, obj_type, obj_subtype, obj_id): # (s) +def add_obj_language(language, obj_type, obj_subtype, obj_id, objs_containers=set()): # (s) if not obj_subtype: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_lang.sadd(f'objs:langs:{obj_type}', language) r_lang.sadd(f'objs:lang:{obj_type}:{obj_subtype}', language) - r_lang.sadd(f'obj:lang:{obj_global_id}', language) + new = r_lang.sadd(f'obj:lang:{obj_global_id}', language) r_lang.sadd(f'languages:{language}', f'{obj_type}:{obj_subtype}') ################### REMOVE ME ??? r_lang.sadd(f'langs:{obj_type}:{obj_subtype}:{language}', obj_global_id) -def remove_obj_language(language, obj_type, obj_subtype, obj_id): + if new: + for global_id in objs_containers: + r_lang.zincrby(f'obj:langs:stat:{global_id}', 1, language) + + +def remove_obj_language(language, obj_type, obj_subtype, obj_id, objs_containers=set()): if not obj_subtype: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' - r_lang.srem(f'obj:lang:{obj_global_id}', language) + rem = r_lang.srem(f'obj:lang:{obj_global_id}', language) delete_obj_translation(obj_global_id, language) @@ -353,27 +361,33 @@ def remove_obj_language(language, obj_type, obj_subtype, obj_id): if r_lang.scard(f'objs:langs:{obj_type}') <= 1: r_lang.srem(f'objs:langs:{obj_type}', language) + if rem: + for global_id in objs_containers: + r = r_lang.zincrby(f'obj:langs:stat:{global_id}', -1, language) + if r < 1: + r_lang.zrem(f'obj:langs:stat:{global_id}', language) + # TODO handle fields -def detect_obj_language(obj_type, obj_subtype, obj_id, content): +def detect_obj_language(obj_type, obj_subtype, obj_id, content, objs_containers=set()): detector = LanguagesDetector(nb_langs=1) language = detector.detect(content) if language: language = language[0] previous_lang = get_obj_languages(obj_type, obj_subtype, obj_id) if previous_lang: - previous_lang = previous_lang[0] + previous_lang = previous_lang.pop() if language != previous_lang: - remove_obj_language(language, obj_type, obj_subtype, obj_id) - add_obj_language(language, obj_type, obj_subtype, obj_id) + remove_obj_language(previous_lang, obj_type, obj_subtype, obj_id, objs_containers=objs_containers) + add_obj_language(language, obj_type, obj_subtype, obj_id, objs_containers=objs_containers) else: - add_obj_language(language, obj_type, obj_subtype, obj_id) + add_obj_language(language, obj_type, obj_subtype, obj_id, objs_containers=objs_containers) return language ## Translation -def _get_obj_translation(obj_global_id, language, field=''): +def r_get_obj_translation(obj_global_id, language, field=''): return r_lang.hget(f'tr:{obj_global_id}:{field}', language) -def get_obj_translation(obj_global_id, language, source=None, content=None, field=''): +def _get_obj_translation(obj_global_id, language, source=None, content=None, field='', objs_containers=set()): """ Returns translated content """ @@ -385,17 +399,20 @@ def get_obj_translation(obj_global_id, language, source=None, content=None, fiel # r_cache.expire(f'translation:{language}:{obj_global_id}:{field}', 0) return translation # TODO HANDLE FIELDS TRANSLATION - translation = _get_obj_translation(obj_global_id, language, field=field) + translation = r_get_obj_translation(obj_global_id, language, field=field) if not translation: source, translation = LanguageTranslator().translate(content, source=source, target=language) if source and translation: obj_type, subtype, obj_id = obj_global_id.split(':', 2) - add_obj_language(source, obj_type, subtype, obj_id) + add_obj_language(source, obj_type, subtype, obj_id, objs_containers=objs_containers) if translation: r_cache.set(f'translation:{language}:{obj_global_id}:{field}', translation) r_cache.expire(f'translation:{language}:{obj_global_id}:{field}', 300) return translation +def get_obj_translation(obj_global_id, language, source=None, content=None, field='', objs_containers=set()): + return _get_obj_translation(obj_global_id, language, source=source, content=content, field=field, objs_containers=objs_containers) + # TODO Force to edit ???? diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 397d5aca..e4b0d82c 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -480,7 +480,6 @@ def api_manually_translate_message(message_id, source, translation_target, trans return {"status": "error", "reason": "Max Size reached"}, 400 all_languages = Language.get_translation_languages() if source not in all_languages: - print(source) return {"status": "error", "reason": "Unknown source Language"}, 400 message_language = message.get_language() if message_language != source: diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 8f1d16d4..6d57d2cf 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -103,8 +103,17 @@ class Message(AbstractObject): return message_id def get_chat_id(self): # TODO optimize -> use me to tag Chat - chat_id = self.get_basename().rsplit('_', 1)[0] - return chat_id + c_id = self.id.split('/') + return c_id[2] + + def get_chat(self): + c_id = self.id.split('/') + return f'chat:{c_id[0]}:{c_id[2]}' + + def get_subchannel(self): + subchannel = self.get_correlation('chat-subchannel') + if subchannel.get('chat-subchannel'): + return f'user-account:{subchannel["chat-subchannel"].pop()}' def get_thread(self): for child in self.get_childrens(): @@ -183,25 +192,6 @@ class Message(AbstractObject): else: return None - def get_translation(self, content=None, source=None, target='fr'): - """ - Returns translated content - """ - - # return self._get_field('translated') - global_id = self.get_global_id() - translation = r_cache.get(f'translation:{target}:{global_id}') - r_cache.expire(f'translation:{target}:{global_id}', 0) - if translation: - return translation - if not content: - content = self.get_content() - translation = Language.LanguageTranslator().translate(content, source=source, target=target) - if translation: - r_cache.set(f'translation:{target}:{global_id}', translation) - r_cache.expire(f'translation:{target}:{global_id}', 300) - return translation - def _set_translation(self, translation): """ Set translated content @@ -305,6 +295,8 @@ class Message(AbstractObject): else: source = None meta['translation'] = self.translate(content=meta.get('content'), source=source, target=translation_target) + if 'language' in options: + meta['language'] = self.get_language() # meta['encoding'] = None return meta @@ -318,11 +310,29 @@ class Message(AbstractObject): # self._set_translation(translated) # return translated - def create(self, content, translation=None, tags=[]): + ## Language ## + + def get_objs_container(self): + objs_containers = set() + # chat + objs_containers.add(self.get_chat()) + subchannel = self.get_subchannel() + if subchannel: + objs_containers.add(subchannel) + # thread = self.get_thread() # TODO Get current thread + # if thread: + # objs_containers.add(thread) + return objs_containers + + #- Language -# + + def create(self, content, language=None, translation=None, tags=[]): self._set_field('content', content) - # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) - if translation: + if not language and content: + language = self.detect_language() + if translation and content: self._set_translation(translation) + self.set_translation(language, translation) for tag in tags: self.add_tag(tag) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 40a114ef..78a0698a 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -25,7 +25,7 @@ from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship -from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_translation, set_obj_translation, delete_obj_translation +from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -302,26 +302,33 @@ class AbstractObject(ABC): ## -Relationship- ## + def get_objs_container(self): + return set() + ## Language ## def get_languages(self): return get_obj_languages(self.type, self.get_subtype(r_str=True), self.id) def add_language(self, language): - return add_obj_language(language, self.type, self.get_subtype(r_str=True), self.id) + return add_obj_language(language, self.type, self.get_subtype(r_str=True), self.id, objs_containers=self.get_objs_container()) def remove_language(self, language): - return remove_obj_language(language, self.type, self.get_subtype(r_str=True), self.id) + return remove_obj_language(language, self.type, self.get_subtype(r_str=True), self.id, objs_containers=self.get_objs_container()) def edit_language(self, old_language, new_language): - self.remove_language(old_language) + if old_language: + self.remove_language(old_language) self.add_language(new_language) def detect_language(self, field=''): - return detect_obj_language(self.type, self.get_subtype(r_str=True), self.id, self.get_content()) + return detect_obj_language(self.type, self.get_subtype(r_str=True), self.id, self.get_content(), objs_containers=self.get_objs_container()) + + def get_obj_language_stats(self): + return get_obj_language_stats(self.type, self.get_subtype(r_str=True), self.id) def get_translation(self, language, field=''): - return get_obj_translation(self.get_global_id(), language, field=field) + return get_obj_translation(self.get_global_id(), language, field=field, objs_containers=self.get_objs_container()) def set_translation(self, language, translation, field=''): return set_obj_translation(self.get_global_id(), language, translation, field=field) @@ -333,7 +340,7 @@ class AbstractObject(ABC): global_id = self.get_global_id() if not content: content = self.get_content() - translation = get_obj_translation(global_id, target, source=source, content=content, field=field) + translation = get_obj_translation(global_id, target, source=source, content=content, field=field, objs_containers=self.get_objs_container()) return translation ## -Language- ## diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index b851a5b1..ad3e3d4c 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -253,7 +253,10 @@ def objects_message_translate(): if resp[1] != 200: return create_json_response(resp[0], resp[1]) else: - return redirect(url_for('chats_explorer.objects_message', id=message_id, target=target)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(url_for('chats_explorer.objects_message', id=message_id, target=target)) @chats_explorer.route("/objects/message/detect/language", methods=['GET']) @login_required @@ -265,7 +268,10 @@ def objects_message_detect_language(): if resp[1] != 200: return create_json_response(resp[0], resp[1]) else: - return redirect(url_for('chats_explorer.objects_message', id=message_id, target=target)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(url_for('chats_explorer.objects_message', id=message_id, target=target)) @chats_explorer.route("/objects/user-account", methods=['GET']) @login_required diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 1a52dc67..96265809 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -102,7 +102,7 @@
-
+ Source: