mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 07:47:17 +00:00
chg: [add Hosts module]
This commit is contained in:
parent
512e61cadc
commit
43c2a27538
3 changed files with 32 additions and 24 deletions
|
@ -53,37 +53,38 @@ class DomClassifier(AbstractModule):
|
|||
|
||||
|
||||
def compute(self, message, r_result=False):
|
||||
item = Item(message)
|
||||
host, id = message.split()
|
||||
|
||||
item_content = item.get_content()
|
||||
item = Item(id)
|
||||
item_basename = item.get_basename()
|
||||
item_date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
try:
|
||||
mimetype = item_basic.get_item_mimetype(item.get_id())
|
||||
|
||||
if mimetype.split('/')[0] == "text":
|
||||
self.c.text(rawtext=item_content)
|
||||
self.c.potentialdomain()
|
||||
self.c.validdomain(passive_dns=True, extended=False)
|
||||
#self.redis_logger.debug(self.c.vdomain)
|
||||
self.c.text(rawtext=host)
|
||||
print(self.c.domain)
|
||||
self.c.validdomain(passive_dns=True, extended=False)
|
||||
#self.redis_logger.debug(self.c.vdomain)
|
||||
|
||||
if self.c.vdomain and d4.is_passive_dns_enabled():
|
||||
for dns_record in self.c.vdomain:
|
||||
self.send_message_to_queue(dns_record)
|
||||
print(self.c.vdomain)
|
||||
print()
|
||||
|
||||
localizeddomains = self.c.include(expression=self.cc_tld)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
|
||||
if self.c.vdomain and d4.is_passive_dns_enabled():
|
||||
for dns_record in self.c.vdomain:
|
||||
self.send_message_to_queue(dns_record)
|
||||
|
||||
localizeddomains = self.c.localizedomain(cc=self.cc)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
|
||||
localizeddomains = self.c.include(expression=self.cc_tld)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
|
||||
|
||||
if r_result:
|
||||
return self.c.vdomain
|
||||
localizeddomains = self.c.localizedomain(cc=self.cc)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
|
||||
|
||||
if r_result:
|
||||
return self.c.vdomain
|
||||
|
||||
except IOError as err:
|
||||
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
|
||||
|
|
|
@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
|
|||
[Indexer]
|
||||
subscribe = Redis_Global
|
||||
|
||||
[DomClassifier]
|
||||
[Hosts]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Host
|
||||
|
||||
[DomClassifier]
|
||||
subscribe = Redis_Host
|
||||
publish = Redis_D4_client
|
||||
|
||||
[D4_client]
|
||||
|
|
|
@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
|
|||
|
||||
def test_module(self):
|
||||
item_id = 'tests/2021/01/01/categ.gz'
|
||||
test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
|
||||
test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
|
||||
|
||||
result = self.module_obj.compute(item_id, r_result=True)
|
||||
print(result)
|
||||
self.assertCountEqual(result, test_categ)
|
||||
|
||||
class Test_Module_CreditCards(unittest.TestCase):
|
||||
|
@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
|
|||
self.module_obj = DomClassifier()
|
||||
|
||||
def test_module(self):
|
||||
test_host = 'foo.be'
|
||||
item_id = 'tests/2021/01/01/domain_classifier.gz'
|
||||
result = self.module_obj.compute(item_id, r_result=True)
|
||||
msg = f'{test_host} {item_id}'
|
||||
result = self.module_obj.compute(msg, r_result=True)
|
||||
self.assertTrue(len(result))
|
||||
|
||||
class Test_Module_Global(unittest.TestCase):
|
||||
|
|
Loading…
Reference in a new issue