chg: [modules] migrate D4_client + PgpDump modules

bin/core/D4_client.py

@@ -13,40 +13,43 @@ a Passive DNS server which can be queried later to search for the Passive DNS re
 import os
 import sys
 import time
-from pubsublogger import publisher
+
 sys.path.append(os.environ['AIL_BIN'])
-from Helper import Process
+##################################
+# Import Project packages
+#################################
+from modules.abstract_module import AbstractModule
+from lib import d4
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+# # TODO: launch me in core screen
-import ConfigLoader
-import d4
-
-# # TODO: lauch me in core screen
 # # TODO: check if already launched in core screen
 
-if __name__ == '__main__':
+class D4Client(AbstractModule):
-    publisher.port = 6380
+    """
-    publisher.channel = "Script"
+        D4Client module for AIL framework
+    """
 
-    config_section = 'D4_client'
+    def __init__(self):
-    p = Process(config_section)
+        super(D4Client, self).__init__()
-    publisher.info("""D4_client is Running""")
 
-    last_refresh = time.time()
+        self.d4_client = d4.create_d4_client()
-    d4_client = d4.create_d4_client()
+        self.last_refresh = time.time()
 
-    while True:
+        # Send module state to logs
-        if last_refresh < d4.get_config_last_update_time():
+        self.redis_logger.info(f'Module {self.module_name} initialized')
-            d4_client = d4.create_d4_client()
+
-            last_refresh = time.time()
+    def compute(self, dns_record):
+        # Refresh D4 Client
+        if self.last_refresh < d4.get_config_last_update_time():
+            self.d4_client = d4.create_d4_client()
+            self.last_refresh = time.time()
             print('D4 Client: config updated')
 
-        dns_record = p.get_from_set()
+        if self.d4_client:
-        if dns_record is None:
-            publisher.debug("Script D4_client is idling 1s")
-            time.sleep(1)
-            continue
-
-        if d4_client:
             # Send DNS Record to D4Server
-            d4_client.send_manual_data(dns_record)
+            self.d4_client.send_manual_data(dns_record)
+
+
+if __name__ == '__main__':
+    module = D4Client()
+    module.run()

bin/modules/Keys.py

@@ -59,7 +59,7 @@ class Keys(AbstractModule):
         item = Item(message)
         content = item.get_content()
 
-        find = False
+        # find = False
         get_pgp_content = False
 
         if KeyEnum.PGP_MESSAGE.value in content:
@@ -68,7 +68,7 @@ class Keys(AbstractModule):
             msg = f'infoleak:automatic-detection="pgp-message";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
             get_pgp_content = True
-            find = True
+            # find = True
 
         if KeyEnum.PGP_PUBLIC_KEY_BLOCK.value in content:
             msg = f'infoleak:automatic-detection="pgp-public-key-block";{item.get_id()}'
@@ -80,12 +80,19 @@ class Keys(AbstractModule):
             self.send_message_to_queue(msg, 'Tags')
             get_pgp_content = True
 
+        if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
+            self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message')
+
+            msg = f'infoleak:automatic-detection="pgp-private-key";{item.get_id()}'
+            self.send_message_to_queue(msg, 'Tags')
+            get_pgp_content = True
+
         if KeyEnum.CERTIFICATE.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has a certificate message')
 
             msg = f'infoleak:automatic-detection="certificate";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.RSA_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message')
@@ -93,7 +100,7 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="rsa-private-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has a private key message')
@@ -101,7 +108,7 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="private-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message')
@@ -109,7 +116,7 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="encrypted-private-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message')
@@ -117,7 +124,7 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message')
@@ -125,7 +132,7 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message')
@@ -133,35 +140,28 @@ class Keys(AbstractModule):
 
             msg = f'infoleak:automatic-detection="vpn-static-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.DSA_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message')
 
             msg = f'infoleak:automatic-detection="dsa-private-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         if KeyEnum.EC_PRIVATE_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has an ec private key message')
 
             msg = f'infoleak:automatic-detection="ec-private-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
-
-        if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
-            self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message')
-
-            msg = f'infoleak:automatic-detection="pgp-private-key";{item.get_id()}'
-            self.send_message_to_queue(msg, 'Tags')
-            find = True
 
         if KeyEnum.PUBLIC_KEY.value in content:
             self.redis_logger.warning(f'{item.get_basename()} has a public key message')
 
             msg = f'infoleak:automatic-detection="public-key";{item.get_id()}'
             self.send_message_to_queue(msg, 'Tags')
-            find = True
+            # find = True
 
         # pgp content
         if get_pgp_content:
@@ -175,6 +175,5 @@ class Keys(AbstractModule):
 
 
 if __name__ == '__main__':
-
     module = Keys()
     module.run()

bin/modules/PgpDump.py

@@ -1,50 +1,73 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-    PgpDum module
+The PgpDump Module
+======================
+
+This module Extract ID from PGP Blocks.
 
-    Extract ID from PGP Blocks
 """
 
+##################################
+# Import External packages
+##################################
 import os
-import re
+import sys
-import time
-import uuid
-import redis
-import signal
-import datetime
 import subprocess
 
-from pubsublogger import publisher
 from bs4 import BeautifulSoup
+from uuid import uuid4
 
-from Helper import Process
+sys.path.append(os.environ['AIL_BIN'])
-from packages import Item
+##################################
-from packages import Paste
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+from lib.objects import Pgps
+from lib.objects.Items import Item
 
-from packages import Pgp
 
-class TimeoutException(Exception):
+class PgpDump(AbstractModule):
-    pass
+    """
+    Cve module for AIL framework
+    """
 
-def timeout_handler(signum, frame):
+    def __init__(self):
-    raise TimeoutException
+        super(PgpDump, self).__init__()
 
-signal.signal(signal.SIGALRM, timeout_handler)
+        # check/create pgpdump queue directory (used for huge pgp blocks)
+        self.pgpdump_dir = os.path.join(os.environ['AIL_HOME'], 'temp', 'pgpdump')
+        if not os.path.isdir(self.pgpdump_dir):
+            os.makedirs(self.pgpdump_dir)
 
-# save pgp message in directory, process one time each day
+        # Regex
-def save_in_file(message, pgp_content):
+        self.reg_user_id = r'User ID - .+'
-    print('save in file')
+        self.reg_key_id = r'Key ID - .+'
-    UUID = str(uuid.uuid4())
+        self.reg_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
-    file_name = os.path.join(pgp_dump_dir_to_process, UUID)
+        self.reg_pgp_private_blocs = r'-----BEGIN PGP PRIVATE KEY BLOCK-----[\s\S]+?-----END PGP PRIVATE KEY BLOCK-----'
-    with open(file_name, 'w') as f:
+        self.reg_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
-        f.write(pgp_content)
+        self.reg_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
-    r_serv_db.sadd('pgpdumb:uuid', '{};{}'.format(UUID, message))
+        self.reg_tool_version = r'\bVersion:.*\n'
+        self.reg_block_comment = r'\bComment:.*\n'
 
-def remove_html(item_content):
+        # Waiting time in seconds between to message processed
+        self.pending_seconds = 1
+
+        # init
+        self.item_id = None
+        self.keys = set()
+        self.private_keys = set()
+        self.names = set()
+        self.mails = set()
+        self.symmetrically_encrypted = False
+
+        # Send module state to logs
+        self.redis_logger.info(f'Module {self.module_name} initialized')
+
+    def remove_html(self, pgp_block):
         try:
-        if bool(BeautifulSoup(item_content, "html.parser").find()):
+            if bool(BeautifulSoup(pgp_block, "html.parser").find()):
-            soup = BeautifulSoup(item_content, 'html.parser')
+                soup = BeautifulSoup(pgp_block, 'html.parser')
                 # kill all script and style elements
                 for script in soup(["script", "style"]):
                     script.extract()  # remove
@@ -53,199 +76,156 @@ def remove_html(item_content):
                 text = soup.get_text()
                 return text
             else:
-            return item_content
+                return pgp_block
         except TypeError:
-        return item_content
+            return pgp_block
 
-def extract_all_id(message, item_content, regex=None, is_file=False):
+    def sanitize_pgp_block(self, pgp_block):
+        # Debug
+        print(pgp_block)
+        print()
+        pgp_block = self.remove_html(pgp_block)
+        # Remove Version
+        versions = self.regex_findall(self.reg_tool_version, self.item_id, pgp_block)
+        for version in versions:
+            pgp_block = pgp_block.replace(version, '')
+        # Remove Comment
+        comments = self.regex_findall(self.reg_block_comment, self.item_id, pgp_block)
+        for comment in comments:
+            pgp_block = pgp_block.replace(comment, '')
+        # Remove Empty Lines
+        pgp_block = [s for s in pgp_block.splitlines() if (s and not s.isspace())]
+        pgp_block[0] = pgp_block[0] + '\n'
+        pgp_block[-1] = '\n' + pgp_block[-1]
+        pgp_block = '\n'.join(pgp_block)
 
-    if is_file:
+        # Debug
-        pgp_packet = get_pgp_packet_file(item_content)
+        print(pgp_block)
-        extract_id_from_output(pgp_packet)
+        print('-------------------------------------------------------------------------')
+        return pgp_block
 
-    else:
+    def get_pgpdump_from_file(self, pgp_block):
-        # max execution time on regex
+        print('Save PGP Block in File')
-        signal.alarm(max_execution_time)
+        file_uuid = str(uuid4())
-        try:
+        filepath = os.path.join(self.pgpdump_dir, file_uuid)
-            pgp_extracted_block = re.findall(regex, item_content)
+        with open(filepath, 'w') as f:
-        except TimeoutException:
+            f.write(pgp_block)
-            pgp_extracted_block = []
+        process1 = subprocess.Popen(['pgpdump', filepath], stdout=subprocess.PIPE)
-            p.incr_module_timeout_statistic() # add encoder type
+        output = process1.communicate()[0].decode()
-            print ("{0} processing timeout".format(paste.p_rel_path))
+        os.remove(filepath)
-        else:
+        return output
-            signal.alarm(0)
 
-        for pgp_to_dump in pgp_extracted_block:
+    def get_pgpdump_from_terminal(self, pgp_block):
-
+        process1 = subprocess.Popen(['echo', '-e', pgp_block], stdout=subprocess.PIPE)
-            pgp_packet = get_pgp_packet(message, pgp_to_dump)
+        process2 = subprocess.Popen(['pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE)
-            extract_id_from_output(pgp_packet)
-
-def get_pgp_packet(message, save_path):
-    save_path = '{}'.format(save_path)
-    # remove Version
-    all_version = re.findall(regex_tool_version, save_path)
-    for version in all_version:
-        save_path = save_path.replace(version, '')
-    # remove comment
-    all_comment= re.findall(regex_block_comment, save_path)
-    for comment in all_comment:
-        save_path = save_path.replace(comment, '')
-    # remove empty line
-    save_path = [s for s in save_path.splitlines() if s]
-    save_path[0] = save_path[0] + '\n'
-    save_path[-1] = '\n' + save_path[-1]
-    save_path = '\n'.join(save_path)
-
-    #print(save_path)
-
-    if len(save_path) > 131072:
-        save_in_file(message, save_path)
-        return ''
-    else:
-        process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE)
-        process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE)
         process1.stdout.close()
         output = process2.communicate()[0]
         try:
             output = output.decode()
         except UnicodeDecodeError:
-            publisher.error('Error PgpDump UnicodeDecodeError: {}'.format(message))
+            self.redis_logger.error(f'Error PgpDump UnicodeDecodeError: {self.item_id}')
             output = ''
         return output
 
-def get_pgp_packet_file(file):
+    def get_pgpdump(self, pgp_block):
-    process1 = subprocess.Popen([ 'pgpdump', file], stdout=subprocess.PIPE)
+        if len(pgp_block) > 131072:
-    output = process1.communicate()[0].decode()
+            return self.get_pgpdump_from_file(pgp_block)
-    return output
-
-def extract_id_from_output(pgp_dump_outpout):
-    all_user_id = set(re.findall(regex_user_id, pgp_dump_outpout))
-    for user_id in all_user_id:
-        # avoid key injection in user_id:
-        pgp_dump_outpout.replace(user_id, '', 1)
-
-        user_id = user_id.replace(user_id_str, '', 1)
-        mail = None
-        if ' <' in user_id:
-            name, mail = user_id.rsplit(' <', 1)
-            mail = mail[:-1]
-            set_name.add(name)
-            set_mail.add(mail)
         else:
-            name = user_id
+            return self.get_pgpdump_from_terminal(pgp_block)
-            set_name.add(name)
 
-    all_key_id = set(re.findall(regex_key_id, pgp_dump_outpout))
+    def extract_id_from_pgpdump_output(self, pgpdump_output):
-    for key_id in all_key_id:
+        if 'Secret Key Packet' in pgpdump_output:
-        key_id = key_id.replace(key_id_str, '', 1)
+            private = True
-        set_key.add(key_id)
+        else:
+            private = False
+        users = self.regex_findall(self.reg_user_id, self.item_id, pgpdump_output)
+        for user in users:
+            # avoid key injection in user_id:
+            pgpdump_output.replace(user, '', 1)
+            user = user.replace('User ID - ', '', 1)
+            if ' <' in user:
+                name, mail = user.rsplit(' <', 1)
+                mail = mail[:-1]
+                self.names.add(name)
+                self.mails.add(mail)
+            else:
+                name = user
+                self.names.add(name)
+
+        keys = self.regex_findall(self.reg_key_id, self.item_id, pgpdump_output)
+        for key_id in keys:
+            key_id = key_id.replace('Key ID - ', '', 1)
+            if key_id != '0x0000000000000000':
+                self.keys.add(key_id)
+                if private:
+                    self.private_keys.add(key_id)
+            else:
+                self.symmetrically_encrypted = True
+                print('symmetrically encrypted')
+
+    def compute(self, message):
+        item = Item(message)
+        self.item_id = item.get_id()
+        content = item.get_content()
+
+        pgp_blocks = []
+        # Public Block
+        for pgp_block in self.regex_findall(self.reg_pgp_public_blocs, self.item_id, content):
+            # content = content.replace(pgp_block, '')
+            pgp_block = self.sanitize_pgp_block(pgp_block)
+            pgp_blocks.append(pgp_block)
+        # Private Block
+        for pgp_block in self.regex_findall(self.reg_pgp_private_blocs, self.item_id, content):
+            # content = content.replace(pgp_block, '')
+            pgp_block = self.sanitize_pgp_block(pgp_block)
+            pgp_blocks.append(pgp_block)
+        # Signature
+        for pgp_block in self.regex_findall(self.reg_pgp_signature, self.item_id, content):
+            # content = content.replace(pgp_block, '')
+            pgp_block = self.sanitize_pgp_block(pgp_block)
+            pgp_blocks.append(pgp_block)
+        # Message
+        for pgp_block in self.regex_findall(self.reg_pgp_message, self.item_id, content):
+            pgp_block = self.sanitize_pgp_block(pgp_block)
+            pgp_blocks.append(pgp_block)
+
+        self.symmetrically_encrypted = False
+        self.keys = set()
+        self.private_keys = set()
+        self.names = set()
+        self.mails = set()
+        for pgp_block in pgp_blocks:
+            pgpdump_output = self.get_pgpdump(pgp_block)
+            self.extract_id_from_pgpdump_output(pgpdump_output)
+
+        if self.keys or self.names or self.mails:
+            print(self.item_id)
+            date = item.get_date()
+            for key in self.keys:
+                pgp = Pgps.Pgp(key, 'key')
+                pgp.add(date, self.item_id)
+                print(f'    key: {key}')
+            for name in self.names:
+                pgp = Pgps.Pgp(name, 'name')
+                pgp.add(date, self.item_id)
+                print(f'    name: {name}')
+            for mail in self.mails:
+                pgp = Pgps.Pgp(mail, 'mail')
+                pgp.add(date, self.item_id)
+                print(f'    mail: {mail}')
+
+            # Keys extracted from PGP PRIVATE KEY BLOCK
+            for key in self.private_keys:
+                pgp = Pgps.Pgp(key, 'key')
+                pgp.add_tag('infoleak:automatic-detection="pgp-private-key"')
+                print(f'    private key: {key}')
+
+        if self.symmetrically_encrypted:
+            msg = f'infoleak:automatic-detection="pgp-symmetric";{self.item_id}'
+            self.send_message_to_queue(msg, 'Tags')
 
 
 if __name__ == '__main__':
-    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    module = PgpDump()
-    # Port of the redis instance used by pubsublogger
+    module.run()
-    publisher.port = 6380
-    # Script is the default channel used for the modules.
-    publisher.channel = 'Script'
 
-    # Section name in bin/packages/modules.cfg
-    #config_section = 'PgpDump'
-    config_section = 'PgpDump'
-
-    # Setup the I/O queues
-    p = Process(config_section)
-
-    r_serv_db = redis.StrictRedis(
-        host=p.config.get("ARDB_DB", "host"),
-        port=p.config.getint("ARDB_DB", "port"),
-        db=p.config.getint("ARDB_DB", "db"),
-        decode_responses=True)
-
-    serv_metadata = redis.StrictRedis(
-        host=p.config.get("ARDB_Metadata", "host"),
-        port=p.config.getint("ARDB_Metadata", "port"),
-        db=p.config.getint("ARDB_Metadata", "db"),
-        decode_responses=True)
-
-    # Sent to the logging a description of the module
-    publisher.info("PgpDump started")
-
-    # check/create pgpdump queue directory (used for huge pgp blocks)
-    pgp_dump_dir_to_process = os.path.join(os.environ['AIL_HOME'], 'temp', 'pgpdump')
-    if not os.path.isdir(pgp_dump_dir_to_process):
-        os.makedirs(pgp_dump_dir_to_process)
-
-    user_id_str = 'User ID - '
-    regex_user_id= '{}.+'.format(user_id_str)
-
-    key_id_str = 'Key ID - '
-    regex_key_id = '{}.+'.format(key_id_str)
-    regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
-    regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
-    regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
-    regex_tool_version = r"\bVersion:.*\n"
-    regex_block_comment = r"\bComment:.*\n"
-
-    re.compile(regex_user_id)
-    re.compile(regex_key_id)
-    re.compile(regex_pgp_public_blocs)
-    re.compile(regex_pgp_signature)
-    re.compile(regex_pgp_message)
-    re.compile(regex_tool_version)
-    re.compile(regex_block_comment)
-
-    max_execution_time = p.config.getint("PgpDump", "max_execution_time")
-
-    # Endless loop getting messages from the input queue
-    while True:
-
-        is_file = False
-        set_key = set()
-        set_name = set()
-        set_mail = set()
-
-        if r_serv_db.scard('pgpdumb:uuid') > 0:
-            res = r_serv_db.spop('pgpdumb:uuid')
-            file_to_process, message = res.split(';', 1)
-            file_to_process = os.path.join(pgp_dump_dir_to_process, file_to_process)
-            date = datetime.datetime.now().strftime("%Y/%m/%d")
-            paste = Paste.Paste(message)
-            date = str(paste._get_p_date())
-            print(message)
-            extract_all_id(message, file_to_process, is_file=True)
-            os.remove(file_to_process)
-
-        else:
-            # Get one message from the input queue
-            message = p.get_from_set()
-
-            if message is None:
-                publisher.debug("{} queue is empty, waiting".format(config_section))
-                time.sleep(1)
-                continue
-
-            print(message)
-            paste = Paste.Paste(message)
-
-            date = str(paste._get_p_date())
-            content = paste.get_p_content()
-            content = remove_html(content)
-
-
-            extract_all_id(message, content, regex_pgp_public_blocs)
-            extract_all_id(message, content, regex_pgp_signature)
-            extract_all_id(message, content, regex_pgp_message)
-
-        item_date = Item.get_item_date(message)
-
-        for key_id in set_key:
-            print(key_id)
-            Pgp.pgp.save_item_correlation('key', key_id, message, item_date)
-
-        for name_id in set_name:
-            print(name_id)
-            Pgp.pgp.save_item_correlation('name', name_id, message, item_date)
-
-        for mail_id in set_mail:
-            print(mail_id)
-            Pgp.pgp.save_item_correlation('mail', mail_id, message, item_date)