AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-26 07:47:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [add Hosts module]

Browse source
This commit is contained in:
Terrtia 2022-05-30 09:03:27 +02:00
parent 512e61cadc
commit 43c2a27538
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 32 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
bin/modules/DomClassifier.py
View file

@ -53,37 +53,38 @@ class DomClassifier(AbstractModule):
def compute(self, message, r_result=False): def compute(self, message, r_result=False):
item = Item(message) host, id = message.split()
item_content = item.get_content() item = Item(id)
item_basename = item.get_basename() item_basename = item.get_basename()
item_date = item.get_date() item_date = item.get_date()
item_source = item.get_source() item_source = item.get_source()
try: try:
mimetype = item_basic.get_item_mimetype(item.get_id())
if mimetype.split('/')[0] == "text": self.c.text(rawtext=host)
self.c.text(rawtext=item_content) print(self.c.domain)
self.c.potentialdomain() self.c.validdomain(passive_dns=True, extended=False)
self.c.validdomain(passive_dns=True, extended=False) #self.redis_logger.debug(self.c.vdomain)
#self.redis_logger.debug(self.c.vdomain)
if self.c.vdomain and d4.is_passive_dns_enabled(): print(self.c.vdomain)
for dns_record in self.c.vdomain: print()
self.send_message_to_queue(dns_record)
localizeddomains = self.c.include(expression=self.cc_tld) if self.c.vdomain and d4.is_passive_dns_enabled():
if localizeddomains: for dns_record in self.c.vdomain:
print(localizeddomains) self.send_message_to_queue(dns_record)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
localizeddomains = self.c.localizedomain(cc=self.cc) localizeddomains = self.c.include(expression=self.cc_tld)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
if r_result: localizeddomains = self.c.localizedomain(cc=self.cc)
return self.c.vdomain if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
if r_result:
return self.c.vdomain
except IOError as err: except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")

6
bin/packages/modules.cfg
View file

@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
[Indexer] [Indexer]
subscribe = Redis_Global subscribe = Redis_Global
[DomClassifier] [Hosts]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Host
[DomClassifier]
subscribe = Redis_Host
publish = Redis_D4_client publish = Redis_D4_client
[D4_client] [D4_client]

7
tests/test_modules.py
View file

@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
def test_module(self): def test_module(self):
item_id = 'tests/2021/01/01/categ.gz' item_id = 'tests/2021/01/01/categ.gz'
test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve'] test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
result = self.module_obj.compute(item_id, r_result=True) result = self.module_obj.compute(item_id, r_result=True)
print(result)
self.assertCountEqual(result, test_categ) self.assertCountEqual(result, test_categ)
class Test_Module_CreditCards(unittest.TestCase): class Test_Module_CreditCards(unittest.TestCase):
@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
self.module_obj = DomClassifier() self.module_obj = DomClassifier()
def test_module(self): def test_module(self):
test_host = 'foo.be'
item_id = 'tests/2021/01/01/domain_classifier.gz' item_id = 'tests/2021/01/01/domain_classifier.gz'
result = self.module_obj.compute(item_id, r_result=True) msg = f'{test_host} {item_id}'
result = self.module_obj.compute(msg, r_result=True)
self.assertTrue(len(result)) self.assertTrue(len(result))
class Test_Module_Global(unittest.TestCase): class Test_Module_Global(unittest.TestCase):