chg: [queues] add new image queue + add exif module

bin/LAUNCH.sh

@@ -273,6 +273,9 @@ function launching_scripts {
     screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x"
     sleep 0.1
 
+    screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x"
+    sleep 0.1
+
     ##################################
     #       TRACKERS MODULES         #
     ##################################

bin/importer/FeederImporter.py

@@ -98,7 +98,8 @@ class FeederImporter(AbstractImporter):
             gzip64_content = feeder.get_gzip64_content()
             return obj, f'{feeder_name} {gzip64_content}'
         else:  # Messages save on DB
-            return obj, f'{feeder_name}'
+            if obj.exists():
+                return obj, f'{feeder_name}'
 
 
 class FeederModuleImporter(AbstractModule):

bin/importer/feeders/abstract_chats_feeder.py

@@ -122,7 +122,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
 
         # TODO sanitize obj type
         obj_type = self.get_obj_type()
-        print(obj_type)
 
         if obj_type == 'image':
             self.obj = Images.Image(self.json_data['data-sha256'])
@@ -226,6 +225,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         """
         # meta = self.get_json_meta()
 
+        objs = set()
+        if self.obj:
+            objs.add(self.obj)
+
         date, timestamp = self.get_message_date_timestamp()
 
         # REPLY
@@ -245,14 +248,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             message_id = self.get_message_id()
             message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
             message = Messages.Message(message_id)
+            # create empty message if message don't exists
+            if not message.exists():
+                message.create('')
+                objs.add(message)
+
             if message.exists():
                 obj = Images.create(self.get_message_content())
                 obj.add(date, message)
                 obj.set_parent(obj_global_id=message.get_global_id())
-            else:
-                obj = None
 
-        if obj:
+        for obj in objs:  # TODO PERF avoid parsing metas multpile times
 
             # CHAT
             chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)

bin/lib/objects/Messages.py

@@ -287,7 +287,7 @@ class Message(AbstractObject):
     #     self._set_translation(translated)
     #     return translated
 
-    def create(self, content, translation, tags):
+    def create(self, content, translation=None, tags=[]):
         self._set_field('content', content)
         # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
         if translation:
@@ -315,7 +315,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
 def create(obj_id, content, translation=None, tags=[]):
     message = Message(obj_id)
     if not message.exists():
-        message.create(content, translation, tags)
+        message.create(content, translation=translation, tags=tags)
     return message
 
 

bin/modules/Exif.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Exif Module
+======================
+
+"""
+
+##################################
+# Import External packages
+##################################
+import os
+import sys
+
+from PIL import Image, ExifTags
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+
+
+class Exif(AbstractModule):
+    """
+    CveModule for AIL framework
+    """
+
+    def __init__(self):
+        super(Exif, self).__init__()
+
+        # Waiting time in seconds between to message processed
+        self.pending_seconds = 1
+
+        # Send module state to logs
+        self.logger.info(f'Module {self.module_name} initialized')
+
+    def compute(self, message):
+        image = self.get_obj()
+        print(image)
+        img = Image.open(image.get_filepath())
+        img_exif = img.getexif()
+        print(img_exif)
+        if img_exif:
+            for key, val in img_exif.items():
+                if key in ExifTags.TAGS:
+                    print(f'{ExifTags.TAGS[key]}:{val}')
+                else:
+                    print(f'{key}:{val}')
+            sys.exit(0)
+
+        # tag = 'infoleak:automatic-detection="cve"'
+        # Send to Tags Queue
+        # self.add_message_to_queue(message=tag, queue='Tags')
+
+
+if __name__ == '__main__':
+
+    module = Exif()
+    module.run()

bin/modules/Global.py

@@ -124,9 +124,11 @@ class Global(AbstractModule):
 
             else:
                 self.logger.info(f"Empty Item: {message} not processed")
-        elif self.obj:
+        elif self.obj.type == 'message':
             # TODO send to specific object queue => image, ...
             self.add_message_to_queue(obj=self.obj, queue='Item')
+        elif self.obj.type == 'image':
+            self.add_message_to_queue(obj=self.obj, queue='Image')
         else:
             self.logger.critical(f"Empty obj: {self.obj} {message} not processed")
 

configs/modules.cfg

@@ -1,3 +1,8 @@
+######## IMPORTERS ########
+
+[Crawler]
+publish = Importers,Tags
+
 [ZMQModuleImporter]
 publish = Importers
 
@@ -13,8 +18,6 @@ publish = Importers
 [PystemonModuleImporter]
 publish = Importers
 
-####################################################
-
 [Mixer]
 subscribe = Importers
 publish = SaveObj
@@ -22,9 +25,13 @@ publish = SaveObj
 [Sync_importer]
 publish = Importers,Tags
 
+######## OBJ SAVER ########
+
 [Global]
 subscribe = SaveObj
-publish = Item
+publish = Item,Image
+
+######## ITEM + MESSAGE ########
 
 [Duplicates]
 subscribe = Duplicate
@@ -40,25 +47,19 @@ publish = Host
 subscribe = Host
 publish = D4_client
 
-[D4Client]
+[Tracker_Typo_Squatting]		# TODO MOVE ME	
-subscribe = D4_client
-
-[Retro_Hunt_Module]
-publish = Tags
-
-[Tracker_Typo_Squatting]
 subscribe = Host
 publish = Tags
 
-[Tracker_Term]
+[Tracker_Term] 				# TODO MOVE ME
 subscribe = Item
 publish = Tags
 
-[Tracker_Regex]
+[Tracker_Regex] 			# TODO MOVE ME
 subscribe = Item
 publish = Tags
 
-[Tracker_Yara]
+[Tracker_Yara] 				# TODO MOVE ME
 subscribe = Item
 publish = Tags
 
@@ -70,7 +71,7 @@ publish = Tags
 subscribe = Item
 publish = Tags
 
-[Languages]
+[Languages]  				# TODO MOVE ME
 subscribe = Item
 
 [Categ]
@@ -106,13 +107,6 @@ publish = Tags
 subscribe = Url
 publish = Tags
 
-[Tags]
-subscribe = Tags
-publish = Tag_feed
-
-[MISP_Thehive_Auto_Push]
-subscribe = Tag_feed
-
 #[SentimentAnalysis]
 #subscribe = Item
 
@@ -151,9 +145,6 @@ publish = Tags
 [SubmitPaste]
 publish = Importers
 
-[Crawler]
-publish = Importers,Tags
-
 [IPAddress]
 subscribe = Item
 publish = Tags
@@ -164,6 +155,32 @@ publish = Tags
 #[Sync_module]
 #publish = Sync
 
+######## IMAGE ########
+
+[Exif]
+subscribe = Image
+publish = Tags
+
+
+######## CORE ########
+
+[Tags]
+subscribe = Tags
+publish = Tag_feed
+
+[Retro_Hunt_Module]
+publish = Tags
+
+######## OTHER ########
+
+[D4Client]
+subscribe = D4_client
+
+[MISP_Thehive_Auto_Push]
+subscribe = Tag_feed
+
+######## DOC ########
+
 # [My_Module_Name]
 # subscribe = Global # Queue name
 # publish = Tags # Queue name