ail-framework/bin/modules/Credential.py

203 lines
8.2 KiB
Python
Raw Normal View History

2018-05-04 11:53:29 +00:00
#!/usr/bin/env python3
2016-02-05 15:15:09 +00:00
# -*-coding:UTF-8 -*
"""
The Credential Module
=====================
This module is consuming the Redis-list created by the Categ module.
2020-05-20 15:03:58 +00:00
It apply credential regexes on item content and warn if above a threshold.
2017-07-18 14:57:15 +00:00
It also split the username and store it into redis for searching purposes.
Redis organization:
uniqNumForUsername: unique number attached to unique username
uniqNumForPath: unique number attached to unique path
2017-07-20 08:24:48 +00:00
-> uniqNum are used to avoid string duplication
2017-07-18 14:57:15 +00:00
AllCredentials: hashed set where keys are username and value are their uniq number
AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username
AllPath: hashed set where keys are path and value are their uniq number
AllPathRev: the opposite of AllPath, uniqNum -> path
CredToPathMapping_uniqNumForUsername -> (set) -> uniqNumForPath
2017-07-18 14:57:15 +00:00
"""
2021-04-28 13:24:33 +00:00
##################################
# Import External packages
##################################
2020-05-20 15:03:58 +00:00
import os
import sys
import time
2016-02-05 15:15:09 +00:00
import re
from datetime import datetime
from pyfaup.faup import Faup
2021-04-28 13:24:33 +00:00
sys.path.append(os.environ['AIL_BIN'])
2021-04-28 13:24:33 +00:00
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib import ConfigLoader
2022-09-08 08:31:57 +00:00
from lib import Statistics
2020-05-04 09:02:24 +00:00
2017-07-18 14:57:15 +00:00
2021-04-28 13:24:33 +00:00
class Credential(AbstractModule):
"""
Credential module for AIL framework
"""
2018-04-16 12:50:04 +00:00
2021-04-28 13:24:33 +00:00
# Split username with spec. char or with upper case, distinguish start with upper
REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+"
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
2021-04-28 13:24:33 +00:00
def __init__(self):
super(Credential, self).__init__()
2020-05-20 15:03:58 +00:00
2021-04-28 13:24:33 +00:00
self.faup = Faup()
self.regex_web = r"((?:https?:\/\/)[\.-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
self.regex_cred = r"[a-zA-Z0-9\\._-]+@[a-zA-Z0-9\\.-]+\.[a-zA-Z]{2,6}[\\rn :\_\-]{1,10}[a-zA-Z0-9\_\-]+"
self.regex_site_for_stats = r"@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:"
2021-04-28 13:24:33 +00:00
# Database
config_loader = ConfigLoader.ConfigLoader()
# self.server_cred = config_loader.get_redis_conn("ARDB_TermCred")
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
2021-04-28 13:24:33 +00:00
# Config values
self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold")
self.criticalNumberToAlert = config_loader.get_config_int("Credential", "criticalNumberToAlert")
self.max_execution_time = 30
# Waiting time in seconds between to message processed
2021-04-28 13:24:33 +00:00
self.pending_seconds = 10
2021-04-28 13:24:33 +00:00
# Send module state to logs
self.redis_logger.info(f"Module {self.module_name} initialized")
2018-04-16 12:50:04 +00:00
2021-04-28 13:24:33 +00:00
def compute(self, message):
2020-05-04 09:11:35 +00:00
item_id, count = message.split()
item = Item(item_id)
item_content = item.get_content()
2018-04-26 12:42:39 +00:00
2022-09-08 08:31:57 +00:00
# TODO: USE SETS
2021-04-28 13:24:33 +00:00
# Extract all credentials
all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content)
if all_credentials:
2021-04-28 13:24:33 +00:00
nb_cred = len(all_credentials)
message = f'Checked {nb_cred} credentials found.'
all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content)
2020-05-20 15:03:58 +00:00
if all_sites:
2021-04-28 13:24:33 +00:00
discovered_sites = ', '.join(all_sites)
message += f' Related websites: {discovered_sites}'
print(message)
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{item.get_id()}'
2021-04-28 13:24:33 +00:00
# num of creds above threshold, publish an alert
2021-04-28 13:24:33 +00:00
if nb_cred > self.criticalNumberToAlert:
print(f"========> Found more than 10 credentials in this file : {item.get_id()}")
2021-04-28 13:24:33 +00:00
self.redis_logger.warning(to_print)
msg = f'infoleak:automatic-detection="credential";{item.get_id()}'
self.send_message_to_queue(msg, 'Tags')
2021-04-28 13:24:33 +00:00
site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content)
2021-04-28 13:24:33 +00:00
creds_sites = {}
for site in site_occurrence:
2021-04-28 13:24:33 +00:00
site_domain = site[1:-1].lower()
if site_domain in creds_sites.keys():
creds_sites[site_domain] += 1
else:
creds_sites[site_domain] = 1
for url in all_sites:
self.faup.decode(url)
domain = self.faup.get()['domain']
# # TODO: # FIXME: remove me, check faup versionb
2021-04-28 13:24:33 +00:00
try:
domain = domain.decode()
except:
pass
if domain in creds_sites.keys():
creds_sites[domain] += 1
else:
creds_sites[domain] = 1
for site, num in creds_sites.items(): # Send for each different site to moduleStats
mssg = f'credential;{num};{site};{item.get_date()}'
print(mssg)
2021-08-18 13:36:05 +00:00
self.send_message_to_queue(mssg, 'ModuleStats')
2021-04-28 13:24:33 +00:00
if all_sites:
discovered_sites = ', '.join(all_sites)
print(f"=======> Probably on : {discovered_sites}")
2021-04-28 13:24:33 +00:00
date = datetime.now().strftime("%Y%m")
2022-09-08 08:31:57 +00:00
nb_tlds = {}
2021-04-28 13:24:33 +00:00
for cred in all_credentials:
maildomains = re.findall(r"@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", cred.lower())[0]
2021-04-28 13:24:33 +00:00
self.faup.decode(maildomains)
tld = self.faup.get()['tld']
# # TODO: # FIXME: remove me
2021-04-28 13:24:33 +00:00
try:
tld = tld.decode()
except:
pass
2022-09-08 08:31:57 +00:00
nb_tlds[tld] = nb_tlds.get(tld, 0) + 1
for tld in nb_tlds:
Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
2021-04-28 13:24:33 +00:00
else:
self.redis_logger.info(to_print)
print(f'found {nb_cred} credentials')
2021-04-28 13:24:33 +00:00
2022-08-19 14:53:31 +00:00
# # TODO: # FIXME: TEMP DESABLE
# # For searching credential in termFreq
# for cred in all_credentials:
# cred = cred.split('@')[0] #Split to ignore mail address
#
# # unique number attached to unique path
# uniq_num_path = self.server_cred.incr(Credential.REDIS_KEY_NUM_PATH)
# self.server_cred.hmset(Credential.REDIS_KEY_ALL_PATH_SET, {item.get_id(): uniq_num_path})
# self.server_cred.hmset(Credential.REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: item.get_id()})
#
# # unique number attached to unique username
# uniq_num_cred = self.server_cred.hget(Credential.REDIS_KEY_ALL_CRED_SET, cred)
# if uniq_num_cred is None:
# # cred do not exist, create new entries
# uniq_num_cred = self.server_cred.incr(Credential.REDIS_KEY_NUM_USERNAME)
# self.server_cred.hmset(Credential.REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
# self.server_cred.hmset(Credential.REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
#
# # Add the mapping between the credential and the path
# self.server_cred.sadd(Credential.REDIS_KEY_MAP_CRED_TO_PATH+'_'+str(uniq_num_cred), uniq_num_path)
#
# # Split credentials on capital letters, numbers, dots and so on
# # Add the split to redis, each split point towards its initial credential unique number
# splitedCred = re.findall(Credential.REGEX_CRED, cred)
# for partCred in splitedCred:
# if len(partCred) > self.minimumLengthThreshold:
# self.server_cred.sadd(partCred, uniq_num_cred)
2021-04-28 13:24:33 +00:00
if __name__ == '__main__':
2021-04-28 13:24:33 +00:00
module = Credential()
module.run()