chg: [queues] add new image queue + add exif module

This commit is contained in:
terrtia 2023-11-15 15:45:25 +01:00
parent 36ff2bb216
commit 2ea5f82760
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
7 changed files with 121 additions and 32 deletions

View file

@ -273,6 +273,9 @@ function launching_scripts {
screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x"
sleep 0.1
##################################
# TRACKERS MODULES #
##################################

View file

@ -98,7 +98,8 @@ class FeederImporter(AbstractImporter):
gzip64_content = feeder.get_gzip64_content()
return obj, f'{feeder_name} {gzip64_content}'
else: # Messages save on DB
return obj, f'{feeder_name}'
if obj.exists():
return obj, f'{feeder_name}'
class FeederModuleImporter(AbstractModule):

View file

@ -122,7 +122,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# TODO sanitize obj type
obj_type = self.get_obj_type()
print(obj_type)
if obj_type == 'image':
self.obj = Images.Image(self.json_data['data-sha256'])
@ -226,6 +225,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
"""
# meta = self.get_json_meta()
objs = set()
if self.obj:
objs.add(self.obj)
date, timestamp = self.get_message_date_timestamp()
# REPLY
@ -245,14 +248,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
message_id = self.get_message_id()
message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
message = Messages.Message(message_id)
# create empty message if message don't exists
if not message.exists():
message.create('')
objs.add(message)
if message.exists():
obj = Images.create(self.get_message_content())
obj.add(date, message)
obj.set_parent(obj_global_id=message.get_global_id())
else:
obj = None
if obj:
for obj in objs: # TODO PERF avoid parsing metas multpile times
# CHAT
chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)

View file

@ -287,7 +287,7 @@ class Message(AbstractObject):
# self._set_translation(translated)
# return translated
def create(self, content, translation, tags):
def create(self, content, translation=None, tags=[]):
self._set_field('content', content)
# r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
if translation:
@ -315,7 +315,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
def create(obj_id, content, translation=None, tags=[]):
message = Message(obj_id)
if not message.exists():
message.create(content, translation, tags)
message.create(content, translation=translation, tags=tags)
return message

60
bin/modules/Exif.py Executable file
View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Exif Module
======================
"""
##################################
# Import External packages
##################################
import os
import sys
from PIL import Image, ExifTags
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
class Exif(AbstractModule):
"""
CveModule for AIL framework
"""
def __init__(self):
super(Exif, self).__init__()
# Waiting time in seconds between to message processed
self.pending_seconds = 1
# Send module state to logs
self.logger.info(f'Module {self.module_name} initialized')
def compute(self, message):
image = self.get_obj()
print(image)
img = Image.open(image.get_filepath())
img_exif = img.getexif()
print(img_exif)
if img_exif:
for key, val in img_exif.items():
if key in ExifTags.TAGS:
print(f'{ExifTags.TAGS[key]}:{val}')
else:
print(f'{key}:{val}')
sys.exit(0)
# tag = 'infoleak:automatic-detection="cve"'
# Send to Tags Queue
# self.add_message_to_queue(message=tag, queue='Tags')
if __name__ == '__main__':
module = Exif()
module.run()

View file

@ -124,9 +124,11 @@ class Global(AbstractModule):
else:
self.logger.info(f"Empty Item: {message} not processed")
elif self.obj:
elif self.obj.type == 'message':
# TODO send to specific object queue => image, ...
self.add_message_to_queue(obj=self.obj, queue='Item')
elif self.obj.type == 'image':
self.add_message_to_queue(obj=self.obj, queue='Image')
else:
self.logger.critical(f"Empty obj: {self.obj} {message} not processed")

View file

@ -1,3 +1,8 @@
######## IMPORTERS ########
[Crawler]
publish = Importers,Tags
[ZMQModuleImporter]
publish = Importers
@ -13,8 +18,6 @@ publish = Importers
[PystemonModuleImporter]
publish = Importers
####################################################
[Mixer]
subscribe = Importers
publish = SaveObj
@ -22,9 +25,13 @@ publish = SaveObj
[Sync_importer]
publish = Importers,Tags
######## OBJ SAVER ########
[Global]
subscribe = SaveObj
publish = Item
publish = Item,Image
######## ITEM + MESSAGE ########
[Duplicates]
subscribe = Duplicate
@ -40,25 +47,19 @@ publish = Host
subscribe = Host
publish = D4_client
[D4Client]
subscribe = D4_client
[Retro_Hunt_Module]
publish = Tags
[Tracker_Typo_Squatting]
[Tracker_Typo_Squatting] # TODO MOVE ME
subscribe = Host
publish = Tags
[Tracker_Term]
[Tracker_Term] # TODO MOVE ME
subscribe = Item
publish = Tags
[Tracker_Regex]
[Tracker_Regex] # TODO MOVE ME
subscribe = Item
publish = Tags
[Tracker_Yara]
[Tracker_Yara] # TODO MOVE ME
subscribe = Item
publish = Tags
@ -70,7 +71,7 @@ publish = Tags
subscribe = Item
publish = Tags
[Languages]
[Languages] # TODO MOVE ME
subscribe = Item
[Categ]
@ -106,13 +107,6 @@ publish = Tags
subscribe = Url
publish = Tags
[Tags]
subscribe = Tags
publish = Tag_feed
[MISP_Thehive_Auto_Push]
subscribe = Tag_feed
#[SentimentAnalysis]
#subscribe = Item
@ -151,9 +145,6 @@ publish = Tags
[SubmitPaste]
publish = Importers
[Crawler]
publish = Importers,Tags
[IPAddress]
subscribe = Item
publish = Tags
@ -164,6 +155,32 @@ publish = Tags
#[Sync_module]
#publish = Sync
######## IMAGE ########
[Exif]
subscribe = Image
publish = Tags
######## CORE ########
[Tags]
subscribe = Tags
publish = Tag_feed
[Retro_Hunt_Module]
publish = Tags
######## OTHER ########
[D4Client]
subscribe = D4_client
[MISP_Thehive_Auto_Push]
subscribe = Tag_feed
######## DOC ########
# [My_Module_Name]
# subscribe = Global # Queue name
# publish = Tags # Queue name