mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [add Hosts module]
This commit is contained in:
parent
512e61cadc
commit
43c2a27538
3 changed files with 32 additions and 24 deletions
|
@ -53,21 +53,22 @@ class DomClassifier(AbstractModule):
|
||||||
|
|
||||||
|
|
||||||
def compute(self, message, r_result=False):
|
def compute(self, message, r_result=False):
|
||||||
item = Item(message)
|
host, id = message.split()
|
||||||
|
|
||||||
item_content = item.get_content()
|
item = Item(id)
|
||||||
item_basename = item.get_basename()
|
item_basename = item.get_basename()
|
||||||
item_date = item.get_date()
|
item_date = item.get_date()
|
||||||
item_source = item.get_source()
|
item_source = item.get_source()
|
||||||
try:
|
try:
|
||||||
mimetype = item_basic.get_item_mimetype(item.get_id())
|
|
||||||
|
|
||||||
if mimetype.split('/')[0] == "text":
|
self.c.text(rawtext=host)
|
||||||
self.c.text(rawtext=item_content)
|
print(self.c.domain)
|
||||||
self.c.potentialdomain()
|
|
||||||
self.c.validdomain(passive_dns=True, extended=False)
|
self.c.validdomain(passive_dns=True, extended=False)
|
||||||
#self.redis_logger.debug(self.c.vdomain)
|
#self.redis_logger.debug(self.c.vdomain)
|
||||||
|
|
||||||
|
print(self.c.vdomain)
|
||||||
|
print()
|
||||||
|
|
||||||
if self.c.vdomain and d4.is_passive_dns_enabled():
|
if self.c.vdomain and d4.is_passive_dns_enabled():
|
||||||
for dns_record in self.c.vdomain:
|
for dns_record in self.c.vdomain:
|
||||||
self.send_message_to_queue(dns_record)
|
self.send_message_to_queue(dns_record)
|
||||||
|
|
|
@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
|
||||||
[Indexer]
|
[Indexer]
|
||||||
subscribe = Redis_Global
|
subscribe = Redis_Global
|
||||||
|
|
||||||
[DomClassifier]
|
[Hosts]
|
||||||
subscribe = Redis_Global
|
subscribe = Redis_Global
|
||||||
|
publish = Redis_Host
|
||||||
|
|
||||||
|
[DomClassifier]
|
||||||
|
subscribe = Redis_Host
|
||||||
publish = Redis_D4_client
|
publish = Redis_D4_client
|
||||||
|
|
||||||
[D4_client]
|
[D4_client]
|
||||||
|
|
|
@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
|
||||||
|
|
||||||
def test_module(self):
|
def test_module(self):
|
||||||
item_id = 'tests/2021/01/01/categ.gz'
|
item_id = 'tests/2021/01/01/categ.gz'
|
||||||
test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
|
test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
|
||||||
|
|
||||||
result = self.module_obj.compute(item_id, r_result=True)
|
result = self.module_obj.compute(item_id, r_result=True)
|
||||||
|
print(result)
|
||||||
self.assertCountEqual(result, test_categ)
|
self.assertCountEqual(result, test_categ)
|
||||||
|
|
||||||
class Test_Module_CreditCards(unittest.TestCase):
|
class Test_Module_CreditCards(unittest.TestCase):
|
||||||
|
@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
|
||||||
self.module_obj = DomClassifier()
|
self.module_obj = DomClassifier()
|
||||||
|
|
||||||
def test_module(self):
|
def test_module(self):
|
||||||
|
test_host = 'foo.be'
|
||||||
item_id = 'tests/2021/01/01/domain_classifier.gz'
|
item_id = 'tests/2021/01/01/domain_classifier.gz'
|
||||||
result = self.module_obj.compute(item_id, r_result=True)
|
msg = f'{test_host} {item_id}'
|
||||||
|
result = self.module_obj.compute(msg, r_result=True)
|
||||||
self.assertTrue(len(result))
|
self.assertTrue(len(result))
|
||||||
|
|
||||||
class Test_Module_Global(unittest.TestCase):
|
class Test_Module_Global(unittest.TestCase):
|
||||||
|
|
Loading…
Reference in a new issue