From 2a45cfd75c549d3840c7d8c69e3d6b3d3a93bd0c Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 15 Jan 2025 11:55:51 +0100 Subject: [PATCH] chg: [retro hunt + update] add ocr retro hunt + new update --- bin/lib/ail_core.py | 2 +- bin/lib/chats_viewer.py | 20 +++++ bin/lib/objects/ail_objects.py | 4 + update/v6.0.1/Update.py | 75 +++++++++++++++++++ update/v6.0.1/Update.sh | 38 ++++++++++ .../templates/hunter/add_retro_hunt_task.html | 4 + 6 files changed, 142 insertions(+), 1 deletion(-) create mode 100755 update/v6.0.1/Update.py create mode 100755 update/v6.0.1/Update.sh diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 7aea993e..98307db7 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -93,7 +93,7 @@ def get_objects_tracked(): return ['barcode', 'decoded', 'item', 'pgp', 'message', 'ocr', 'qrcode', 'title'] def get_objects_retro_hunted(): - return ['decoded', 'item', 'message'] + return ['decoded', 'item', 'message', 'ocr'] def get_all_objects_with_subtypes_tuple(): str_objs = [] diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index f937eb74..e5ee233f 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -25,6 +25,7 @@ from lib.objects import ChatThreads from lib.objects import Messages from lib.objects.BarCodes import Barcode from lib.objects.QrCodes import Qrcode +from lib.objects.Ocrs import Ocr from lib.objects import UsersAccount from lib.objects import Usernames from lib import Language @@ -421,6 +422,25 @@ def get_nb_messages_iterator(filters={}): nb_messages += chat.get_nb_messages() return nb_messages + +def get_ocrs_iterator(filters={}): + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + print(chat.get_correlation('ocr')) + for ocr in chat.get_correlation('ocr').get('ocr', []): + _, ocr_id = ocr.split(':', 1) + yield Ocr(ocr_id) + +def get_nb_ors_iterator(filters={}): + nb = 0 + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + nb += chat.get_nb_correlation('ocr') + return nb + + def get_chat_object_messages_meta(c_messages): temp_chats = {} for date in c_messages: diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index b6a6c505..3489ba1d 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -371,6 +371,8 @@ def obj_iterator(obj_type, filters): return Pgps.get_all_pgps_objects(filters=filters) elif obj_type == 'message': return chats_viewer.get_messages_iterator(filters=filters) + elif obj_type == 'ocr': + return chats_viewer.get_ocrs_iterator(filters=filters) elif obj_type == 'title': return Titles.Titles().get_iterator() @@ -390,6 +392,8 @@ def card_obj_iterator(obj_type, filters): return Pgps.nb_all_pgps_objects(filters=filters) elif obj_type == 'message': return chats_viewer.get_nb_messages_iterator(filters=filters) + elif obj_type == 'ocr': + return chats_viewer.get_nb_ors_iterator(filters=filters) def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ diff --git a/update/v6.0.1/Update.py b/update/v6.0.1/Update.py new file mode 100755 index 00000000..59af6fed --- /dev/null +++ b/update/v6.0.1/Update.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.ail_updater import AIL_Updater +from lib import ail_users +from lib import crawlers +from lib import Investigations +from lib import Tracker + +def _fix_user_lowercase(): + r_tracking = Investigations.r_tracking + r_tracker = Tracker.r_tracker + r_crawler = crawlers.r_crawler + + for user_id in ail_users.get_users(): + l_user_id = user_id.lower() + if user_id != l_user_id: + print(f'Updating {user_id} ...') + ail_users.kill_session_user(user_id) + + # Investigations + for investigation_uuid in Investigations.get_user_all_investigations(user_id): + r_tracking.srem(f'investigations:user:{user_id}', investigation_uuid) + r_tracking.sadd(f'investigations:user:{l_user_id}', investigation_uuid) + r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', l_user_id) + + # Trackers + for tracker_uuid in Tracker.get_user_trackers(user_id): + tracker = Tracker.Tracker(tracker_uuid) + tracker_type = tracker.get_type() + + r_tracker.rename(f'user:tracker:{user_id}:{tracker_type}', f'user:tracker:{l_user_id}:{tracker_type}') + r_tracker.rename(f'user:tracker:{user_id}', f'user:tracker:{l_user_id}') + + # creator + r_tracker.hset(f'tracker:{tracker_uuid}', 'user_id', l_user_id) + + try: + r_tracker.rename(f'trackers:user:{user_id}', f'trackers:user:{l_user_id}') + except Exception: + pass + + # Cookiejar + for cookiejar_uuid in crawlers.get_cookiejars_user(user_id): + cookiejar = crawlers.Cookiejar(cookiejar_uuid) + # creator + cookiejar._set_user(l_user_id) + + try: + r_crawler.rename(f'cookiejars:user:{user_id}', f'cookiejars:user:{l_user_id}') + except Exception: + pass + + # ail_user + ail_users._fix_user_lowercase(user_id) + + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) + + +if __name__ == '__main__': + _fix_user_lowercase() + # updater = Updater('v6.0.1') + # updater.run_update() diff --git a/update/v6.0.1/Update.sh b/update/v6.0.1/Update.sh new file mode 100755 index 00000000..69f36350 --- /dev/null +++ b/update/v6.0.1/Update.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +echo -e $GREEN"Updating UI resources..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ut +wait + +echo -e $GREEN"Updating python requirement..."$DEFAULT +pip install -U flask-sock + +# SUBMODULES # +git submodule update + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v6.0/Update.py +wait +echo "" +echo "" + +exit 0 diff --git a/var/www/templates/hunter/add_retro_hunt_task.html b/var/www/templates/hunter/add_retro_hunt_task.html index 83190d79..db430144 100644 --- a/var/www/templates/hunter/add_retro_hunt_task.html +++ b/var/www/templates/hunter/add_retro_hunt_task.html @@ -82,6 +82,10 @@ +
+ + +
{#
#} {# #} {# #}