chg: [add Hosts module]

bin/modules/DomClassifier.py

@@ -53,21 +53,22 @@ class DomClassifier(AbstractModule):
 
 
     def compute(self, message, r_result=False):
-        item = Item(message)
+        host, id = message.split()
 
-        item_content = item.get_content()
+        item = Item(id)
         item_basename = item.get_basename()
         item_date = item.get_date()
         item_source = item.get_source()
         try:
-            mimetype = item_basic.get_item_mimetype(item.get_id())
 
-            if mimetype.split('/')[0] == "text":
-                self.c.text(rawtext=item_content)
-                self.c.potentialdomain()
+            self.c.text(rawtext=host)
+            print(self.c.domain)
             self.c.validdomain(passive_dns=True, extended=False)
             #self.redis_logger.debug(self.c.vdomain)
 
+            print(self.c.vdomain)
+            print()
+
             if self.c.vdomain and d4.is_passive_dns_enabled():
                 for dns_record in self.c.vdomain:
                     self.send_message_to_queue(dns_record)

bin/packages/modules.cfg

@@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
 [Indexer]
 subscribe = Redis_Global
 
-[DomClassifier]
+[Hosts]
 subscribe = Redis_Global
+publish = Redis_Host
+
+[DomClassifier]
+subscribe = Redis_Host
 publish = Redis_D4_client
 
 [D4_client]

tests/test_modules.py

@@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
 
     def test_module(self):
         item_id = 'tests/2021/01/01/categ.gz'
-        test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
+        test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
 
         result = self.module_obj.compute(item_id, r_result=True)
+        print(result)
         self.assertCountEqual(result, test_categ)
 
 class Test_Module_CreditCards(unittest.TestCase):
@@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
         self.module_obj = DomClassifier()
 
     def test_module(self):
+        test_host = 'foo.be'
         item_id = 'tests/2021/01/01/domain_classifier.gz'
-        result = self.module_obj.compute(item_id, r_result=True)
+        msg = f'{test_host} {item_id}'
+        result = self.module_obj.compute(msg, r_result=True)
         self.assertTrue(len(result))
 
 class Test_Module_Global(unittest.TestCase):