chg: [retro hunt + update] add ocr retro hunt + new update

bin/lib/ail_core.py

@@ -93,7 +93,7 @@ def get_objects_tracked():
     return ['barcode', 'decoded', 'item', 'pgp', 'message', 'ocr', 'qrcode', 'title']
 
 def get_objects_retro_hunted():
-    return ['decoded', 'item', 'message']
+    return ['decoded', 'item', 'message', 'ocr']
 
 def get_all_objects_with_subtypes_tuple():
     str_objs = []

bin/lib/chats_viewer.py

@@ -25,6 +25,7 @@ from lib.objects import ChatThreads
 from lib.objects import Messages
 from lib.objects.BarCodes import Barcode
 from lib.objects.QrCodes import Qrcode
+from lib.objects.Ocrs import Ocr
 from lib.objects import UsersAccount
 from lib.objects import Usernames
 from lib import Language
@@ -421,6 +422,25 @@ def get_nb_messages_iterator(filters={}):
             nb_messages += chat.get_nb_messages()
     return nb_messages
 
+
+def get_ocrs_iterator(filters={}):
+    for instance_uuid in get_chat_service_instances():
+        for chat_id in ChatServiceInstance(instance_uuid).get_chats():
+            chat = Chats.Chat(chat_id, instance_uuid)
+            print(chat.get_correlation('ocr'))
+            for ocr in chat.get_correlation('ocr').get('ocr', []):
+                _, ocr_id = ocr.split(':', 1)
+                yield Ocr(ocr_id)
+
+def get_nb_ors_iterator(filters={}):
+    nb = 0
+    for instance_uuid in get_chat_service_instances():
+        for chat_id in ChatServiceInstance(instance_uuid).get_chats():
+            chat = Chats.Chat(chat_id, instance_uuid)
+            nb += chat.get_nb_correlation('ocr')
+    return nb
+
+
 def get_chat_object_messages_meta(c_messages):
     temp_chats = {}
     for date in c_messages:

bin/lib/objects/ail_objects.py

@@ -371,6 +371,8 @@ def obj_iterator(obj_type, filters):
         return Pgps.get_all_pgps_objects(filters=filters)
     elif obj_type == 'message':
         return chats_viewer.get_messages_iterator(filters=filters)
+    elif obj_type == 'ocr':
+        return chats_viewer.get_ocrs_iterator(filters=filters)
     elif obj_type == 'title':
         return Titles.Titles().get_iterator()
 
@@ -390,6 +392,8 @@ def card_obj_iterator(obj_type, filters):
         return Pgps.nb_all_pgps_objects(filters=filters)
     elif obj_type == 'message':
         return chats_viewer.get_nb_messages_iterator(filters=filters)
+    elif obj_type == 'ocr':
+        return chats_viewer.get_nb_ors_iterator(filters=filters)
 
 def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
     """

update/v6.0.1/Update.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+sys.path.append(os.environ['AIL_HOME'])
+##################################
+# Import Project packages
+##################################
+from update.bin.ail_updater import AIL_Updater
+from lib import ail_users
+from lib import crawlers
+from lib import Investigations
+from lib import Tracker
+
+def _fix_user_lowercase():
+    r_tracking = Investigations.r_tracking
+    r_tracker = Tracker.r_tracker
+    r_crawler = crawlers.r_crawler
+
+    for user_id in ail_users.get_users():
+        l_user_id = user_id.lower()
+        if user_id != l_user_id:
+            print(f'Updating {user_id} ...')
+            ail_users.kill_session_user(user_id)
+
+            # Investigations
+            for investigation_uuid in Investigations.get_user_all_investigations(user_id):
+                r_tracking.srem(f'investigations:user:{user_id}', investigation_uuid)
+                r_tracking.sadd(f'investigations:user:{l_user_id}', investigation_uuid)
+                r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', l_user_id)
+
+            # Trackers
+            for tracker_uuid in Tracker.get_user_trackers(user_id):
+                tracker = Tracker.Tracker(tracker_uuid)
+                tracker_type = tracker.get_type()
+
+                r_tracker.rename(f'user:tracker:{user_id}:{tracker_type}', f'user:tracker:{l_user_id}:{tracker_type}')
+                r_tracker.rename(f'user:tracker:{user_id}', f'user:tracker:{l_user_id}')
+
+                # creator
+                r_tracker.hset(f'tracker:{tracker_uuid}', 'user_id', l_user_id)
+
+            try:
+                r_tracker.rename(f'trackers:user:{user_id}', f'trackers:user:{l_user_id}')
+            except Exception:
+                pass
+
+            # Cookiejar
+            for cookiejar_uuid in crawlers.get_cookiejars_user(user_id):
+                cookiejar = crawlers.Cookiejar(cookiejar_uuid)
+                # creator
+                cookiejar._set_user(l_user_id)
+
+            try:
+                r_crawler.rename(f'cookiejars:user:{user_id}', f'cookiejars:user:{l_user_id}')
+            except Exception:
+                pass
+
+            # ail_user
+            ail_users._fix_user_lowercase(user_id)
+
+
+class Updater(AIL_Updater):
+    """default Updater."""
+
+    def __init__(self, version):
+        super(Updater, self).__init__(version)
+
+
+if __name__ == '__main__':
+    _fix_user_lowercase()
+    # updater = Updater('v6.0.1')
+    # updater.run_update()

update/v6.0.1/Update.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+echo -e $GREEN"Updating UI resources..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ut
+wait
+
+echo -e $GREEN"Updating python requirement..."$DEFAULT
+pip install -U flask-sock
+
+# SUBMODULES #
+git submodule update
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v6.0/Update.py
+wait
+echo ""
+echo ""
+
+exit 0

var/www/templates/hunter/add_retro_hunt_task.html

@@ -82,6 +82,10 @@
                                                 <input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
                                                 <label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i>&nbsp;Messages</label>
                                             </div>
+                                            <div class="custom-control custom-switch mt-1">
+                                                <input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" checked="">
+                                                <label class="custom-control-label" for="ocr_obj"><i class="fas fa-expand"></i>&nbsp;Ocrs</label>
+                                            </div>
 {#                                                    <div class="custom-control custom-switch mt-1">#}
 {#                                                        <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
 {#                                                        <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}