ail-framework/bin/modules/Categ.py

125 lines
3.9 KiB
Python
Raw Normal View History

2018-05-04 11:53:29 +00:00
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The ZMQ_PubSub_Categ Module
============================
Each words files created under /files/ are representing categories.
This modules take these files and compare them to
the content of an obj.
When a word from a obj match one or more of these words file, the filename of
the obj / the obj id is published/forwarded to the next modules.
Each category (each files) are representing a dynamic channel.
This mean that if you create 1000 files under /files/ you'll have 1000 channels
where every time there is a matching word to a category, the obj containing
this word will be pushed to this specific channel.
..note:: The channel will have the name of the file created.
Implementing modules can start here, create your own category file,
and then create your own module to treat the specific paste matching this
category.
Requirements
------------
*Need running Redis instances. (Redis)
*Categories files of words in /files/ need to be created
"""
2021-04-02 07:52:05 +00:00
##################################
# Import External packages
##################################
import argparse
import os
2014-09-05 15:05:45 +00:00
import re
import sys
2021-04-02 07:52:05 +00:00
sys.path.append(os.environ['AIL_BIN'])
2021-04-02 07:52:05 +00:00
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
2014-08-19 17:07:07 +00:00
2021-04-02 07:52:05 +00:00
class Categ(AbstractModule):
"""
Categ module for AIL framework
"""
def __init__(self, categ_files_dir=os.path.join(os.environ['AIL_HOME'], 'files')):
2021-04-02 07:52:05 +00:00
"""
Init Categ
"""
super(Categ, self).__init__()
2021-05-19 14:57:20 +00:00
self.categ_files_dir = categ_files_dir
config_loader = ConfigLoader()
2021-05-19 14:57:20 +00:00
# default = 1 string
self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
2021-05-19 14:57:20 +00:00
self.reload_categ_words()
2021-04-02 07:52:05 +00:00
self.redis_logger.info("Script Categ started")
2021-05-19 14:57:20 +00:00
# # TODO: trigger reload on change ( save last reload time, ...)
def reload_categ_words(self):
2021-06-17 16:16:31 +00:00
categories = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve', 'ApiKey']
2021-04-02 07:52:05 +00:00
tmp_dict = {}
for filename in categories:
bname = os.path.basename(filename)
tmp_dict[bname] = []
2021-05-19 14:57:20 +00:00
with open(os.path.join(self.categ_files_dir, filename), 'r') as f:
2021-04-02 07:52:05 +00:00
patterns = [r'%s' % ( re.escape(s.strip()) ) for s in f]
tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE)
2021-05-19 14:57:20 +00:00
self.categ_words = tmp_dict.items()
2021-05-19 14:57:20 +00:00
def compute(self, message, r_result=False):
# Get obj Object
obj = self.get_obj()
# Get obj content
content = obj.get_content()
2021-05-19 14:57:20 +00:00
categ_found = []
# Search for pattern categories in obj content
2021-05-19 14:57:20 +00:00
for categ, pattern in self.categ_words:
2021-04-02 07:52:05 +00:00
if obj.type == 'message' or obj.type == 'ocr':
self.add_message_to_queue(message='0', queue=categ)
else:
2018-04-16 12:50:04 +00:00
found = set(re.findall(pattern, content))
lenfound = len(found)
if lenfound >= self.matchingThreshold:
categ_found.append(categ)
msg = str(lenfound)
# Export message to categ queue
print(msg, categ)
self.add_message_to_queue(message=msg, queue=categ)
self.redis_logger.debug(
f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
2016-02-10 15:39:56 +00:00
2021-05-19 14:57:20 +00:00
if r_result:
return categ_found
2021-04-02 07:52:05 +00:00
2023-04-21 12:53:33 +00:00
2021-05-19 14:57:20 +00:00
if __name__ == '__main__':
2021-04-02 07:52:05 +00:00
2021-05-19 14:57:20 +00:00
# SCRIPT PARSER #
parser = argparse.ArgumentParser(description='Start Categ module on files.')
parser.add_argument(
'-d', type=str, default=os.path.join(os.environ['AIL_HOME'], 'files'),
2021-05-19 14:57:20 +00:00
help='Path to the directory containing the category files.',
action='store')
args = parser.parse_args()
2021-04-02 07:52:05 +00:00
2021-05-19 14:57:20 +00:00
module = Categ(categ_files_dir=args.d)
2021-04-02 07:52:05 +00:00
module.run()