ail-framework/bin/modules/Hosts.py

73 lines
2.2 KiB
Python
Raw Normal View History

2022-05-30 07:15:50 +00:00
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Hosts Module
======================
This module is consuming the Redis-list created by the Global module.
It is looking for Hosts
"""
##################################
# Import External packages
##################################
import os
import re
import sys
import DomainClassifier.domainclassifier
2022-05-30 07:15:50 +00:00
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
class Hosts(AbstractModule):
"""
Hosts module for AIL framework
"""
def __init__(self):
super(Hosts, self).__init__()
config_loader = ConfigLoader()
self.r_cache = config_loader.get_redis_conn("Redis_Cache")
# regex timeout
self.regex_timeout = 30
# Waiting time in seconds between to message processed
2022-05-30 07:15:50 +00:00
self.pending_seconds = 1
redis_host = config_loader.get_config_str('Redis_Cache', 'host')
redis_port = config_loader.get_config_int('Redis_Cache', 'port')
redis_db = config_loader.get_config_int('Redis_Cache', 'db')
self.dom_classifier = DomainClassifier.domainclassifier.Extract(rawtext="",
redis_host=redis_host,
redis_port=redis_port,
redis_db=redis_db,
re_timeout=30)
2023-05-12 13:29:53 +00:00
self.logger.info(f"Module: {self.module_name} Launched")
2022-05-30 07:15:50 +00:00
def compute(self, message):
obj = self.get_obj()
2022-05-30 07:15:50 +00:00
content = obj.get_content()
self.dom_classifier.text(content)
if self.dom_classifier.domain:
print(f'{len(self.dom_classifier.domain)} host {obj.get_id()}')
# print(self.dom_classifier.domain)
for domain in self.dom_classifier.domain:
if domain:
self.add_message_to_queue(message=domain, queue='Host')
2022-05-30 07:15:50 +00:00
if __name__ == '__main__':
module = Hosts()
module.run()