mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [queues] add new image queue + add exif module
This commit is contained in:
parent
36ff2bb216
commit
2ea5f82760
7 changed files with 121 additions and 32 deletions
|
@ -273,6 +273,9 @@ function launching_scripts {
|
|||
screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
##################################
|
||||
# TRACKERS MODULES #
|
||||
##################################
|
||||
|
|
|
@ -98,7 +98,8 @@ class FeederImporter(AbstractImporter):
|
|||
gzip64_content = feeder.get_gzip64_content()
|
||||
return obj, f'{feeder_name} {gzip64_content}'
|
||||
else: # Messages save on DB
|
||||
return obj, f'{feeder_name}'
|
||||
if obj.exists():
|
||||
return obj, f'{feeder_name}'
|
||||
|
||||
|
||||
class FeederModuleImporter(AbstractModule):
|
||||
|
|
|
@ -122,7 +122,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
|
||||
# TODO sanitize obj type
|
||||
obj_type = self.get_obj_type()
|
||||
print(obj_type)
|
||||
|
||||
if obj_type == 'image':
|
||||
self.obj = Images.Image(self.json_data['data-sha256'])
|
||||
|
@ -226,6 +225,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
"""
|
||||
# meta = self.get_json_meta()
|
||||
|
||||
objs = set()
|
||||
if self.obj:
|
||||
objs.add(self.obj)
|
||||
|
||||
date, timestamp = self.get_message_date_timestamp()
|
||||
|
||||
# REPLY
|
||||
|
@ -245,14 +248,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
message_id = self.get_message_id()
|
||||
message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
|
||||
message = Messages.Message(message_id)
|
||||
# create empty message if message don't exists
|
||||
if not message.exists():
|
||||
message.create('')
|
||||
objs.add(message)
|
||||
|
||||
if message.exists():
|
||||
obj = Images.create(self.get_message_content())
|
||||
obj.add(date, message)
|
||||
obj.set_parent(obj_global_id=message.get_global_id())
|
||||
else:
|
||||
obj = None
|
||||
|
||||
if obj:
|
||||
for obj in objs: # TODO PERF avoid parsing metas multpile times
|
||||
|
||||
# CHAT
|
||||
chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
|
||||
|
|
|
@ -287,7 +287,7 @@ class Message(AbstractObject):
|
|||
# self._set_translation(translated)
|
||||
# return translated
|
||||
|
||||
def create(self, content, translation, tags):
|
||||
def create(self, content, translation=None, tags=[]):
|
||||
self._set_field('content', content)
|
||||
# r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
|
||||
if translation:
|
||||
|
@ -315,7 +315,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
|
|||
def create(obj_id, content, translation=None, tags=[]):
|
||||
message = Message(obj_id)
|
||||
if not message.exists():
|
||||
message.create(content, translation, tags)
|
||||
message.create(content, translation=translation, tags=tags)
|
||||
return message
|
||||
|
||||
|
||||
|
|
60
bin/modules/Exif.py
Executable file
60
bin/modules/Exif.py
Executable file
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Exif Module
|
||||
======================
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ExifTags
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
|
||||
|
||||
class Exif(AbstractModule):
|
||||
"""
|
||||
CveModule for AIL framework
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(Exif, self).__init__()
|
||||
|
||||
# Waiting time in seconds between to message processed
|
||||
self.pending_seconds = 1
|
||||
|
||||
# Send module state to logs
|
||||
self.logger.info(f'Module {self.module_name} initialized')
|
||||
|
||||
def compute(self, message):
|
||||
image = self.get_obj()
|
||||
print(image)
|
||||
img = Image.open(image.get_filepath())
|
||||
img_exif = img.getexif()
|
||||
print(img_exif)
|
||||
if img_exif:
|
||||
for key, val in img_exif.items():
|
||||
if key in ExifTags.TAGS:
|
||||
print(f'{ExifTags.TAGS[key]}:{val}')
|
||||
else:
|
||||
print(f'{key}:{val}')
|
||||
sys.exit(0)
|
||||
|
||||
# tag = 'infoleak:automatic-detection="cve"'
|
||||
# Send to Tags Queue
|
||||
# self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module = Exif()
|
||||
module.run()
|
|
@ -124,9 +124,11 @@ class Global(AbstractModule):
|
|||
|
||||
else:
|
||||
self.logger.info(f"Empty Item: {message} not processed")
|
||||
elif self.obj:
|
||||
elif self.obj.type == 'message':
|
||||
# TODO send to specific object queue => image, ...
|
||||
self.add_message_to_queue(obj=self.obj, queue='Item')
|
||||
elif self.obj.type == 'image':
|
||||
self.add_message_to_queue(obj=self.obj, queue='Image')
|
||||
else:
|
||||
self.logger.critical(f"Empty obj: {self.obj} {message} not processed")
|
||||
|
||||
|
|
|
@ -1,3 +1,8 @@
|
|||
######## IMPORTERS ########
|
||||
|
||||
[Crawler]
|
||||
publish = Importers,Tags
|
||||
|
||||
[ZMQModuleImporter]
|
||||
publish = Importers
|
||||
|
||||
|
@ -13,8 +18,6 @@ publish = Importers
|
|||
[PystemonModuleImporter]
|
||||
publish = Importers
|
||||
|
||||
####################################################
|
||||
|
||||
[Mixer]
|
||||
subscribe = Importers
|
||||
publish = SaveObj
|
||||
|
@ -22,9 +25,13 @@ publish = SaveObj
|
|||
[Sync_importer]
|
||||
publish = Importers,Tags
|
||||
|
||||
######## OBJ SAVER ########
|
||||
|
||||
[Global]
|
||||
subscribe = SaveObj
|
||||
publish = Item
|
||||
publish = Item,Image
|
||||
|
||||
######## ITEM + MESSAGE ########
|
||||
|
||||
[Duplicates]
|
||||
subscribe = Duplicate
|
||||
|
@ -40,25 +47,19 @@ publish = Host
|
|||
subscribe = Host
|
||||
publish = D4_client
|
||||
|
||||
[D4Client]
|
||||
subscribe = D4_client
|
||||
|
||||
[Retro_Hunt_Module]
|
||||
publish = Tags
|
||||
|
||||
[Tracker_Typo_Squatting]
|
||||
[Tracker_Typo_Squatting] # TODO MOVE ME
|
||||
subscribe = Host
|
||||
publish = Tags
|
||||
|
||||
[Tracker_Term]
|
||||
[Tracker_Term] # TODO MOVE ME
|
||||
subscribe = Item
|
||||
publish = Tags
|
||||
|
||||
[Tracker_Regex]
|
||||
[Tracker_Regex] # TODO MOVE ME
|
||||
subscribe = Item
|
||||
publish = Tags
|
||||
|
||||
[Tracker_Yara]
|
||||
[Tracker_Yara] # TODO MOVE ME
|
||||
subscribe = Item
|
||||
publish = Tags
|
||||
|
||||
|
@ -70,7 +71,7 @@ publish = Tags
|
|||
subscribe = Item
|
||||
publish = Tags
|
||||
|
||||
[Languages]
|
||||
[Languages] # TODO MOVE ME
|
||||
subscribe = Item
|
||||
|
||||
[Categ]
|
||||
|
@ -106,13 +107,6 @@ publish = Tags
|
|||
subscribe = Url
|
||||
publish = Tags
|
||||
|
||||
[Tags]
|
||||
subscribe = Tags
|
||||
publish = Tag_feed
|
||||
|
||||
[MISP_Thehive_Auto_Push]
|
||||
subscribe = Tag_feed
|
||||
|
||||
#[SentimentAnalysis]
|
||||
#subscribe = Item
|
||||
|
||||
|
@ -151,9 +145,6 @@ publish = Tags
|
|||
[SubmitPaste]
|
||||
publish = Importers
|
||||
|
||||
[Crawler]
|
||||
publish = Importers,Tags
|
||||
|
||||
[IPAddress]
|
||||
subscribe = Item
|
||||
publish = Tags
|
||||
|
@ -164,6 +155,32 @@ publish = Tags
|
|||
#[Sync_module]
|
||||
#publish = Sync
|
||||
|
||||
######## IMAGE ########
|
||||
|
||||
[Exif]
|
||||
subscribe = Image
|
||||
publish = Tags
|
||||
|
||||
|
||||
######## CORE ########
|
||||
|
||||
[Tags]
|
||||
subscribe = Tags
|
||||
publish = Tag_feed
|
||||
|
||||
[Retro_Hunt_Module]
|
||||
publish = Tags
|
||||
|
||||
######## OTHER ########
|
||||
|
||||
[D4Client]
|
||||
subscribe = D4_client
|
||||
|
||||
[MISP_Thehive_Auto_Push]
|
||||
subscribe = Tag_feed
|
||||
|
||||
######## DOC ########
|
||||
|
||||
# [My_Module_Name]
|
||||
# subscribe = Global # Queue name
|
||||
# publish = Tags # Queue name
|
||||
|
|
Loading…
Reference in a new issue