fix: [modules] fix modules

bin/LAUNCH.sh

@@ -217,7 +217,7 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cryptocurrencies.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cve.py; read x"
+    screen -S "Script_AIL" -X screen -t "CveModule" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CveModule.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
     sleep 0.1

bin/exporter/MailExporter.py

@@ -60,9 +60,6 @@ class MailExporter(AbstractExporter, ABC):
                 self.port is None):
             raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
 
-    def import(self):
-        pass
-
     def get_smtp_client(self):
         # try:
         if self.pw is not None:

bin/lib/Tracker.py

@@ -26,7 +26,9 @@ from lib.Users import User
 config_loader = ConfigLoader.ConfigLoader()
 r_cache = config_loader.get_redis_conn("Redis_Cache")
 
-r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
+r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
+
+r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") # TODO REMOVE ME
 
 items_dir = config_loader.get_config_str("Directories", "pastes")
 if items_dir[-1] == '/':
@@ -68,12 +70,112 @@ def is_valid_mail(email):
 def verify_mail_list(mail_list):
     for mail in mail_list:
         if not is_valid_mail(mail):
-            return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
+            return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
     return None
 
 ##-- UTILS --##
 ###############
 
+################################################################################################
+################################################################################################
+################################################################################################
+
+class Tracker:
+    def __init__(self, tracker_uuid):
+        self.uuid = tracker_uuid
+
+    def get_uuid(self):
+        return self.uuid
+
+    def exists(self):
+        return r_tracker.exists(f'tracker:{self.uuid}')
+
+    def get_date(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'date')
+
+    def get_first_seen(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'first_seen')
+
+    def get_last_seen(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'last_seen')
+
+    def get_description(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'description')
+
+    def get_level(self):
+        level = r_tracker.hget(f'tracker:{self.uuid}', 'level')
+        if not level:
+            level = 0
+        return int(level)
+
+    def get_sources(self):
+        return r_tracker.smembers(f'tracker:sources:{self.uuid}')
+
+    def get_tracker(self):
+        return r_serv_tracker.hget(f'tracker:{self.uuid}', 'tracked')
+
+    def get_type(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'type')
+
+    def get_tags(self):
+        return r_tracker.smembers(f'tracker:tags:{self.uuid}')
+
+    def mail_export(self):
+        return r_tracker.exists(f'tracker:mail:{self.uuid}')
+
+    def get_mails(self):
+        return r_tracker.smembers(f'tracker:mail:{self.uuid}')
+
+    def get_user(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'user_id')
+
+    def webhook_export(self):
+        return r_tracker.hexists(f'tracker:mail:{self.uuid}', 'webhook')
+
+    def get_webhook(self):
+        return r_tracker.hget(f'tracker:{self.uuid}', 'webhook')
+
+    # TODO get objects/ tracked items
+
+
+    # TODO sparkline
+    def get_meta(self, options):
+        if not options:
+            options = set()
+        meta = {'uuid': self.uuid,
+                'tracker': self.get_tracker(),
+                'type': self.get_type(),
+                'date': self.get_date(),
+                'first_seen': self.get_first_seen(),
+                'last_seen': self.get_last_seen()}
+        if 'user' in options:
+            meta['user'] = self.get_user()
+        if 'level' in options:
+            meta['level'] = self.get_level()
+        if 'description' in options:
+            meta['description'] = self.get_description()
+        if 'tags' in options:
+            meta['tags'] = self.get_tags()
+        if 'sources' in options:
+            meta['sources'] = self.get_sources()
+        if 'mails' in options:
+            meta['mails'] = self.get_mails()
+        if 'webhooks' in options:
+            meta['webhook'] = self.get_webhook()
+        # if 'sparkline' in options:
+        #     meta['sparkline'] = get_tracker_sparkline(tracker_uuid)
+
+
+
+
+    # TODO
+    def add(self, obj_id):
+        pass
+
+################################################################################################
+################################################################################################
+################################################################################################
+
 def get_all_tracker_type():
     return ['word', 'set', 'regex', 'yara']
 
@@ -345,7 +447,7 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
 
 ## API ##
 def api_check_tracker_uuid(tracker_uuid):
-    if not is_valid_uuid_v4(task_uuid):
+    if not is_valid_uuid_v4(tracker_uuid):
         return {"status": "error", "reason": "Invalid uuid"}, 400
     if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
         return {"status": "error", "reason": "Unknown uuid"}, 404
@@ -678,6 +780,10 @@ def reload_yara_rules():
         l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
         for tracker_uuid in l_tracker_uuid:
             rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
+    for tracker_uuid in rule_dict:
+        if not os.path.isfile(rule_dict[tracker_uuid]):
+            # TODO IGNORE + LOGS
+            raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists")
     rules = yara.compile(filepaths=rule_dict)
     return rules
 

bin/lib/d4.py

@@ -40,7 +40,7 @@ def is_passive_dns_enabled(cache=True):
         res = r_cache.get('d4:passivedns:enabled')
         if res is None:
             res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
-            r_cache.set('d4:passivedns:enabled', res)
+            r_cache.set('d4:passivedns:enabled', str(res))
             return res
         else:
             return res == 'True'

bin/lib/objects/Decodeds.py

@@ -3,6 +3,7 @@
 
 import os
 import sys
+import magic
 import requests
 import zipfile
 
@@ -135,9 +136,9 @@ class Decoded(AbstractDaterangeObject):
         obj.first_seen = self.get_first_seen()
         obj.last_seen = self.get_last_seen()
 
-        obj_attrs.append( obj.add_attribute('sha1', value=self.id))
+        obj_attrs.append(obj.add_attribute('sha1', value=self.id))
-        obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype()))
+        obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype()))
-        obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
+        obj_attrs.append(obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
         for obj_attr in obj_attrs:
             for tag in self.get_tags():
                 obj_attr.add_tag(tag)
@@ -177,7 +178,10 @@ class Decoded(AbstractDaterangeObject):
         else:
             return {}
 
+    # TODO
     def guess_mimetype(self, bytes_content):
+        # if not bytes_content:
+        #     bytes_content = self.get_content()
         return magic.from_buffer(bytes_content, mime=True)
 
     # avoid counting the same hash multiple time on the same item
@@ -226,15 +230,15 @@ class Decoded(AbstractDaterangeObject):
         # -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}')
 
     # # TODO: ADD items
-    def create(self, content, date, mimetype=None):
+    # def create(self, content, date, mimetype=None):
-        if not mimetype:
+    #     if not mimetype:
-            mimetype = self.guess_mimetype(content)
+    #         mimetype = self.guess_mimetype(content)
-        self.save_file(content, mimetype)
+    #     self.save_file(content, mimetype)
-
+    #
-
+    #
-        update_decoded_daterange(sha1_string, date_from)
+    #     update_decoded_daterange(sha1_string, date_from)
-        if date_from != date_to and date_to:
+    #     if date_from != date_to and date_to:
-            update_decoded_daterange(sha1_string, date_to)
+    #         update_decoded_daterange(sha1_string, date_to)
 
         #######################################################################################
         #######################################################################################
@@ -266,6 +270,8 @@ class Decoded(AbstractDaterangeObject):
             # file in queue
             elif response_code == -2:
                 report = 'In Queue - Refresh'
+            else:
+                report = 'Error - Unknown VT response'
             self.set_vt_report(report)
             print(json_response)
             print(response_code)

bin/modules/ApiKey.py

@@ -39,6 +39,7 @@ class ApiKey(AbstractModule):
         re.compile(self.re_aws_secret_key)
 
         # r'=AIza[0-9a-zA-Z-_]{35}' keep equal ????
+        # AIza[0-9A-Za-z\\-_]{35}
         self.re_google_api_key = r'AIza[0-9a-zA-Z-_]{35}'
         re.compile(self.re_google_api_key)
 
@@ -80,9 +81,6 @@ class ApiKey(AbstractModule):
             msg = f'infoleak:automatic-detection="api-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
 
-            # Send to duplicate
-            self.send_message_to_queue(item.get_id(), 'Duplicate')
-
             if r_result:
                 return google_api_key, aws_access_key, aws_secret_key
 

bin/modules/LibInjection.py

@@ -76,9 +76,6 @@ class LibInjection(AbstractModule):
             to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
             self.redis_logger.warning(to_print)
 
-            # Send to duplicate
-            self.send_message_to_queue(item_id, 'Duplicate')
-
             # Add tag
             msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
             self.send_message_to_queue(msg, 'Tags')

bin/modules/Phone.py

@@ -56,9 +56,6 @@ class Phone(AbstractModule):
             msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
 
-            # Send to duplicate
-            self.send_message_to_queue(item.get_id(), 'Duplicate')
-
             stats = {}
             for phone_number in results:
                 try:

bin/modules/SQLInjectionDetection.py

@@ -57,9 +57,6 @@ class SQLInjectionDetection(AbstractModule):
             to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
             self.redis_logger.warning(to_print)
 
-            # Send to duplicate
-            self.send_message_to_queue(item_id, 'Duplicate')
-
             # Tag
             msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
             self.send_message_to_queue(msg, 'Tags')

bin/packages/modules.cfg

@@ -36,7 +36,7 @@ publish = Redis_Host
 subscribe = Redis_Host
 publish = Redis_D4_client
 
-[D4_client]
+[D4Client]
 subscribe = Redis_D4_client
 
 [Retro_Hunt]