mirror of
https://github.com/ail-project/ail-framework.git
synced 2025-01-18 08:26:15 +00:00
chg: [pubsublogger] remove old redis_logger
This commit is contained in:
terrtia
parent
feabe9f5df
commit
849977f0c2
32 changed files with 112 additions and 241 deletions
|
|
@ -615,14 +615,13 @@ function launch_all {
|
|||
update;
|
||||
launch_redis;
|
||||
launch_kvrocks;
|
||||
launch_logs;
|
||||
launch_scripts;
|
||||
launch_flask;
|
||||
}
|
||||
|
||||
function menu_display {
|
||||
|
||||
options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
|
||||
options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
|
||||
|
||||
menu() {
|
||||
echo "What do you want to Launch?:"
|
||||
|
|
@ -653,9 +652,6 @@ function menu_display {
|
|||
Kvrocks)
|
||||
launch_kvrocks;
|
||||
;;
|
||||
Logs)
|
||||
launch_logs;
|
||||
;;
|
||||
Scripts)
|
||||
launch_scripts;
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -88,7 +88,6 @@ class Sync_module(AbstractModule):
|
|||
if self.last_refresh_queues < time.time():
|
||||
timeout_processed_objs()
|
||||
self.last_refresh_queues = time.time() + 120
|
||||
self.redis_logger.debug('Timeout queues')
|
||||
# print('Timeout queues')
|
||||
|
||||
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
|
||||
class ApiKey(AbstractModule):
|
||||
"""ApiKey module for AIL framework"""
|
||||
|
|
@ -48,20 +47,19 @@ class ApiKey(AbstractModule):
|
|||
|
||||
def compute(self, message, r_result=False):
|
||||
score = message
|
||||
item = self.get_obj()
|
||||
item_content = item.get_content()
|
||||
obj = self.get_obj()
|
||||
content = obj.get_content()
|
||||
|
||||
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
|
||||
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
|
||||
google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True)
|
||||
aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True)
|
||||
if aws_access_key:
|
||||
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
|
||||
aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True)
|
||||
|
||||
if aws_access_key or google_api_key:
|
||||
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
to_print = obj.get_global_id()
|
||||
|
||||
if google_api_key:
|
||||
print(f'found google api key: {to_print}')
|
||||
self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
|
||||
|
||||
tag = 'infoleak:automatic-detection="google-api-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
@ -69,10 +67,8 @@ class ApiKey(AbstractModule):
|
|||
# # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
|
||||
if aws_access_key:
|
||||
print(f'found AWS key: {to_print}')
|
||||
self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
|
||||
if aws_secret_key:
|
||||
print(f'found AWS secret key')
|
||||
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
|
||||
|
||||
tag = 'infoleak:automatic-detection="aws-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class Categ(AbstractModule):
|
|||
self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
|
||||
|
||||
self.reload_categ_words()
|
||||
self.redis_logger.info("Script Categ started")
|
||||
self.logger.info("Script Categ started")
|
||||
|
||||
# # TODO: trigger reload on change ( save last reload time, ...)
|
||||
def reload_categ_words(self):
|
||||
|
|
@ -103,9 +103,6 @@ class Categ(AbstractModule):
|
|||
print(msg, categ)
|
||||
self.add_message_to_queue(message=msg, queue=categ)
|
||||
|
||||
self.redis_logger.debug(
|
||||
f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
|
||||
|
||||
if r_result:
|
||||
return categ_found
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ Redis organization:
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
|
|
@ -37,9 +36,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib import ConfigLoader
|
||||
# from lib import Statistics
|
||||
|
||||
|
||||
class Credential(AbstractModule):
|
||||
|
|
@ -80,40 +77,36 @@ class Credential(AbstractModule):
|
|||
self.pending_seconds = 10
|
||||
|
||||
# Send module state to logs
|
||||
self.redis_logger.info(f"Module {self.module_name} initialized")
|
||||
self.logger.info(f"Module {self.module_name} initialized")
|
||||
|
||||
def compute(self, message):
|
||||
|
||||
count = message
|
||||
item = self.get_obj()
|
||||
obj = self.get_obj()
|
||||
|
||||
item_content = item.get_content()
|
||||
content = obj.get_content()
|
||||
|
||||
# TODO: USE SETS
|
||||
# Extract all credentials
|
||||
all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content)
|
||||
all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content)
|
||||
if all_credentials:
|
||||
nb_cred = len(all_credentials)
|
||||
message = f'Checked {nb_cred} credentials found.'
|
||||
|
||||
all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True)
|
||||
all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True)
|
||||
if all_sites:
|
||||
discovered_sites = ', '.join(all_sites)
|
||||
message += f' Related websites: {discovered_sites}'
|
||||
|
||||
print(message)
|
||||
|
||||
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
|
||||
|
||||
# num of creds above threshold, publish an alert
|
||||
if nb_cred > self.criticalNumberToAlert:
|
||||
print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
|
||||
self.redis_logger.warning(to_print)
|
||||
|
||||
tag = 'infoleak:automatic-detection="credential"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
||||
site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content)
|
||||
site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content)
|
||||
|
||||
creds_sites = {}
|
||||
|
||||
|
|
@ -162,8 +155,7 @@ class Credential(AbstractModule):
|
|||
# for tld in nb_tlds:
|
||||
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
|
||||
else:
|
||||
self.redis_logger.info(to_print)
|
||||
print(f'found {nb_cred} credentials')
|
||||
print(f'found {nb_cred} credentials {self.obj.get_global_id()}')
|
||||
|
||||
# # TODO: # FIXME: TEMP DESABLE
|
||||
# # For searching credential in termFreq
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ The CreditCards Module
|
|||
|
||||
This module is consuming the Redis-list created by the Categ module.
|
||||
|
||||
It apply credit card regexes on item content and warn if a valid card number is found.
|
||||
It apply credit card regexes on object content and warn if a valid card number is found.
|
||||
|
||||
"""
|
||||
|
||||
|
|
@ -23,7 +23,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from packages import lib_refine
|
||||
|
||||
class CreditCards(AbstractModule):
|
||||
|
|
@ -68,10 +67,9 @@ class CreditCards(AbstractModule):
|
|||
return extracted
|
||||
|
||||
def compute(self, message, r_result=False):
|
||||
score = message
|
||||
item = self.get_obj()
|
||||
content = item.get_content()
|
||||
all_cards = self.regex_findall(self.regex, item.id, content)
|
||||
obj = self.get_obj()
|
||||
content = obj.get_content()
|
||||
all_cards = self.regex_findall(self.regex, obj.id, content)
|
||||
|
||||
if len(all_cards) > 0:
|
||||
# self.logger.debug(f'All matching {all_cards}')
|
||||
|
|
@ -84,11 +82,8 @@ class CreditCards(AbstractModule):
|
|||
creditcard_set.add(valid_card)
|
||||
|
||||
# print(creditcard_set)
|
||||
to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
if creditcard_set:
|
||||
mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}'
|
||||
print(mess)
|
||||
self.redis_logger.warning(mess)
|
||||
print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}')
|
||||
|
||||
tag = 'infoleak:automatic-detection="credit-card"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
@ -96,7 +91,7 @@ class CreditCards(AbstractModule):
|
|||
if r_result:
|
||||
return creditcard_set
|
||||
else:
|
||||
self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}')
|
||||
print(f'CreditCard related;{self.obj.get_global_id()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.CryptoCurrencies import CryptoCurrency
|
||||
from lib.objects.Items import Item
|
||||
|
||||
##################################
|
||||
##################################
|
||||
|
|
@ -151,17 +150,10 @@ class Cryptocurrencies(AbstractModule, ABC):
|
|||
|
||||
# debug
|
||||
print(private_keys)
|
||||
to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(),
|
||||
item.get_date(),
|
||||
item.get_basename())
|
||||
self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
|
||||
to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
|
||||
else:
|
||||
private_keys = []
|
||||
|
||||
to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys"
|
||||
print(to_print)
|
||||
self.redis_logger.warning(to_print)
|
||||
print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ class CveModule(AbstractModule):
|
|||
self.logger.info(f'Module {self.module_name} initialized')
|
||||
|
||||
def compute(self, message):
|
||||
count = message
|
||||
item = self.get_obj()
|
||||
item_id = item.get_id()
|
||||
|
||||
|
|
@ -56,9 +55,7 @@ class CveModule(AbstractModule):
|
|||
cve = Cves.Cve(cve_id)
|
||||
cve.add(date, item)
|
||||
|
||||
warning = f'{self.obj.get_global_id()} contains CVEs {cves}'
|
||||
print(warning)
|
||||
self.redis_logger.warning(warning)
|
||||
print(f'{self.obj.get_global_id()} contains CVEs {cves}')
|
||||
|
||||
tag = 'infoleak:automatic-detection="cve"'
|
||||
# Send to Tags Queue
|
||||
|
|
|
|||
|
|
@ -58,10 +58,6 @@ class DomClassifier(AbstractModule):
|
|||
def compute(self, message, r_result=False):
|
||||
host = message
|
||||
|
||||
item = self.get_obj()
|
||||
item_basename = item.get_basename()
|
||||
item_date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
try:
|
||||
|
||||
self.dom_classifier.text(rawtext=host)
|
||||
|
|
@ -82,19 +78,19 @@ class DomClassifier(AbstractModule):
|
|||
localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
|
||||
self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
|
||||
|
||||
if self.cc:
|
||||
localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
|
||||
self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
|
||||
|
||||
if r_result:
|
||||
return self.dom_classifier.vdomain
|
||||
|
||||
except IOError as err:
|
||||
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
|
||||
self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed")
|
||||
raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import Duplicate
|
||||
from lib.objects.Items import Item
|
||||
|
||||
|
||||
class Duplicates(AbstractModule):
|
||||
|
|
@ -92,11 +91,11 @@ class Duplicates(AbstractModule):
|
|||
Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
|
||||
|
||||
if nb_duplicates:
|
||||
self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}')
|
||||
self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}')
|
||||
|
||||
y = time.time()
|
||||
print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
|
||||
# self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x))
|
||||
# self.logger.debug('{}Processed in {} sec'.format(to_print, y-x))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -74,7 +74,6 @@ class Global(AbstractModule):
|
|||
if int(difftime) > 30:
|
||||
to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s'
|
||||
print(to_print)
|
||||
self.redis_logger.debug(to_print)
|
||||
|
||||
self.time_last_stats = time.time()
|
||||
self.processed_item = 0
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import regex_helper
|
||||
|
||||
# TODO REWRITE ME -> PERF + IPV6 + Tracker ?
|
||||
|
|
@ -66,11 +65,11 @@ class IPAddress(AbstractModule):
|
|||
if not self.ip_networks:
|
||||
return None
|
||||
|
||||
item = self.get_obj()
|
||||
content = item.get_content()
|
||||
obj = self.get_obj()
|
||||
content = obj.get_content()
|
||||
|
||||
# list of the regex results in the Item
|
||||
results = self.regex_findall(self.re_ipv4, item.get_id(), content)
|
||||
results = self.regex_findall(self.re_ipv4, obj.get_id(), content)
|
||||
results = set(results)
|
||||
matching_ips = []
|
||||
for ip in results:
|
||||
|
|
@ -83,7 +82,6 @@ class IPAddress(AbstractModule):
|
|||
|
||||
if len(matching_ips) > 0:
|
||||
self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
|
||||
|
||||
# Tag message with IP
|
||||
tag = 'infoleak:automatic-detection="ip"'
|
||||
|
|
@ -93,4 +91,3 @@ class IPAddress(AbstractModule):
|
|||
if __name__ == "__main__":
|
||||
module = IPAddress()
|
||||
module.run()
|
||||
# module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')
|
||||
|
|
|
|||
|
|
@ -24,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
# from lib.ConfigLoader import ConfigLoader
|
||||
# from lib import Statistics
|
||||
|
||||
class Iban(AbstractModule):
|
||||
|
|
@ -40,7 +39,7 @@ class Iban(AbstractModule):
|
|||
def __init__(self, queue=True):
|
||||
super(Iban, self).__init__(queue=queue)
|
||||
|
||||
# Waiting time in secondes between to message proccessed
|
||||
# Waiting time in seconds between to message processed
|
||||
self.pending_seconds = 10
|
||||
|
||||
self.regex_timeout = 30
|
||||
|
|
@ -49,7 +48,7 @@ class Iban(AbstractModule):
|
|||
self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
|
||||
|
||||
# Send module state to logs
|
||||
self.redis_logger.info(f'Module {self.module_name} initialized')
|
||||
self.logger.info(f'Module {self.module_name} initialized')
|
||||
|
||||
def get_iban_number(self, iban):
|
||||
return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN)
|
||||
|
|
@ -73,29 +72,27 @@ class Iban(AbstractModule):
|
|||
return extracted
|
||||
|
||||
def compute(self, message):
|
||||
item = self.get_obj()
|
||||
item_id = item.get_id()
|
||||
obj = self.get_obj()
|
||||
obj_id = obj.get_id()
|
||||
|
||||
ibans = self.regex_findall(self.iban_regex, item_id, item.get_content())
|
||||
ibans = self.regex_findall(self.iban_regex, obj_id, obj.get_content())
|
||||
if ibans:
|
||||
valid_ibans = set()
|
||||
for iban in ibans:
|
||||
iban = iban[1:-1].replace("'", "").split(',')
|
||||
iban = iban[0]+iban[1]+iban[2]
|
||||
iban = ''.join(e for e in iban if e.isalnum())
|
||||
if self.regex_findall(self.iban_regex_verify, item_id, iban):
|
||||
if self.regex_findall(self.iban_regex_verify, obj_id, iban):
|
||||
print(f'checking {iban}')
|
||||
if self.is_valid_iban(iban):
|
||||
valid_ibans.add(iban)
|
||||
|
||||
if valid_ibans:
|
||||
print(f'{valid_ibans} ibans {item_id}')
|
||||
date = datetime.datetime.now().strftime("%Y%m")
|
||||
print(f'{valid_ibans} ibans {self.obj.get_global_id()}')
|
||||
# date = datetime.datetime.now().strftime("%Y%m")
|
||||
# for iban in valid_ibans:
|
||||
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
|
||||
|
||||
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}')
|
||||
# Tags
|
||||
tag = 'infoleak:automatic-detection="iban"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
|
||||
|
||||
class KeyEnum(Enum):
|
||||
|
|
@ -56,14 +55,14 @@ class Keys(AbstractModule):
|
|||
self.pending_seconds = 1
|
||||
|
||||
def compute(self, message):
|
||||
item = self.get_obj()
|
||||
content = item.get_content()
|
||||
obj = self.get_obj()
|
||||
content = obj.get_content()
|
||||
|
||||
# find = False
|
||||
get_pgp_content = False
|
||||
|
||||
if KeyEnum.PGP_MESSAGE.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message')
|
||||
print(f'{self.obj.get_global_id()} has a PGP enc message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="pgp-message"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
@ -81,21 +80,21 @@ class Keys(AbstractModule):
|
|||
get_pgp_content = True
|
||||
|
||||
if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message')
|
||||
print(f'{self.obj.get_global_id()} has a pgp private key block message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="pgp-private-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
get_pgp_content = True
|
||||
|
||||
if KeyEnum.CERTIFICATE.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message')
|
||||
print(f'{self.obj.get_global_id()} has a certificate message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="certificate"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
# find = True
|
||||
|
||||
if KeyEnum.RSA_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message')
|
||||
print(f'{self.obj.get_global_id()} has a RSA private key message')
|
||||
print('rsa private key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="rsa-private-key"'
|
||||
|
|
@ -103,7 +102,7 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message')
|
||||
print(f'{self.obj.get_global_id()} has a private key message')
|
||||
print('private key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="private-key"'
|
||||
|
|
@ -111,7 +110,7 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message')
|
||||
print(f'{self.obj.get_global_id()} has an encrypted private key message')
|
||||
print('encrypted private key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="encrypted-private-key"'
|
||||
|
|
@ -119,7 +118,7 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
|
||||
print(f'{self.obj.get_global_id()} has an openssh private key message')
|
||||
print('openssh private key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="private-ssh-key"'
|
||||
|
|
@ -127,7 +126,7 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message')
|
||||
print(f'{self.obj.get_global_id()} has an ssh2 private key message')
|
||||
print('SSH2 private key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="private-ssh-key"'
|
||||
|
|
@ -135,7 +134,7 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
|
||||
print(f'{self.obj.get_global_id()} has an openssh private key message')
|
||||
print('OpenVPN Static key message found')
|
||||
|
||||
tag = 'infoleak:automatic-detection="vpn-static-key"'
|
||||
|
|
@ -143,21 +142,21 @@ class Keys(AbstractModule):
|
|||
# find = True
|
||||
|
||||
if KeyEnum.DSA_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message')
|
||||
print(f'{self.obj.get_global_id()} has a dsa private key message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="dsa-private-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
# find = True
|
||||
|
||||
if KeyEnum.EC_PRIVATE_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message')
|
||||
print(f'{self.obj.get_global_id()} has an ec private key message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="ec-private-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
# find = True
|
||||
|
||||
if KeyEnum.PUBLIC_KEY.value in content:
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message')
|
||||
print(f'{self.obj.get_global_id()} has a public key message')
|
||||
|
||||
tag = 'infoleak:automatic-detection="public-key"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
|
|
|||
|
|
@ -34,10 +34,9 @@ class LibInjection(AbstractModule):
|
|||
|
||||
self.faup = Faup()
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
item = self.get_obj()
|
||||
url = message
|
||||
|
||||
self.faup.decode(url)
|
||||
|
|
@ -66,12 +65,9 @@ class LibInjection(AbstractModule):
|
|||
# print(f'query is sqli : {result_query}')
|
||||
|
||||
if result_path['sqli'] is True or result_query['sqli'] is True:
|
||||
item_id = item.get_id()
|
||||
print(f"Detected (libinjection) SQL in URL: {item_id}")
|
||||
print(unquote(url))
|
||||
|
||||
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
|
||||
self.redis_logger.warning(to_print)
|
||||
self.logger.info(f'Detected SQL in URL;{self.obj.get_global_id()}')
|
||||
print(unquote(url))
|
||||
|
||||
# Add tag
|
||||
tag = 'infoleak:automatic-detection="sql-injection"'
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.exceptions import MISPConnectionError
|
||||
from lib.objects.Items import Item
|
||||
from lib import Tag
|
||||
from exporter.MISPExporter import MISPExporterAutoDaily
|
||||
from exporter.TheHiveExporter import TheHiveExporterAlertTag
|
||||
|
|
@ -43,7 +42,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
|
|||
if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0:
|
||||
self.tags = Tag.refresh_auto_push()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.info('Tags Auto Push refreshed')
|
||||
self.logger.debug('Tags Auto Push refreshed')
|
||||
|
||||
tag = message
|
||||
item = self.get_obj()
|
||||
|
|
|
|||
|
|
@ -95,20 +95,20 @@ class Mail(AbstractModule):
|
|||
# print()
|
||||
|
||||
except dns.resolver.NoNameservers:
|
||||
self.redis_logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
|
||||
self.logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
|
||||
print('NoNameserver, No non-broken nameservers are available to answer the query.')
|
||||
except dns.resolver.NoAnswer:
|
||||
self.redis_logger.debug('NoAnswer, The response did not contain an answer to the question.')
|
||||
self.logger.debug('NoAnswer, The response did not contain an answer to the question.')
|
||||
print('NoAnswer, The response did not contain an answer to the question.')
|
||||
except dns.name.EmptyLabel:
|
||||
self.redis_logger.debug('SyntaxError: EmptyLabel')
|
||||
self.logger.debug('SyntaxError: EmptyLabel')
|
||||
print('SyntaxError: EmptyLabel')
|
||||
except dns.resolver.NXDOMAIN:
|
||||
# save_mxdomain_in_cache(mxdomain)
|
||||
self.redis_logger.debug('The query name does not exist.')
|
||||
self.logger.debug('The query name does not exist.')
|
||||
print('The query name does not exist.')
|
||||
except dns.name.LabelTooLong:
|
||||
self.redis_logger.debug('The Label is too long')
|
||||
self.logger.debug('The Label is too long')
|
||||
print('The Label is too long')
|
||||
except dns.exception.Timeout:
|
||||
print('dns timeout')
|
||||
|
|
@ -134,9 +134,7 @@ class Mail(AbstractModule):
|
|||
|
||||
# # TODO: sanitize mails
|
||||
def compute(self, message):
|
||||
score = message
|
||||
item = self.get_obj()
|
||||
item_date = item.get_date()
|
||||
|
||||
mails = self.regex_findall(self.email_regex, item.id, item.get_content())
|
||||
mxdomains_email = {}
|
||||
|
|
@ -171,15 +169,15 @@ class Mail(AbstractModule):
|
|||
# for tld in mx_tlds:
|
||||
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
|
||||
|
||||
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
|
||||
msg = f'Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
|
||||
if num_valid_email > self.mail_threshold:
|
||||
print(f'{item.id} Checked {num_valid_email} e-mail(s)')
|
||||
self.redis_logger.warning(msg)
|
||||
self.logger.info(msg)
|
||||
# Tags
|
||||
tag = 'infoleak:automatic-detection="mail"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
elif num_valid_email > 0:
|
||||
self.redis_logger.info(msg)
|
||||
self.logger.info(msg)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
The Onion Module
|
||||
============================
|
||||
|
||||
This module extract url from item and returning only ones which are tor
|
||||
This module extract url from object and returning only ones which are tor
|
||||
related (.onion). All These urls are send to the crawler discovery queue.
|
||||
|
||||
Requirements
|
||||
|
|
@ -69,12 +69,11 @@ class Onion(AbstractModule):
|
|||
onion_urls = []
|
||||
domains = []
|
||||
|
||||
score = message
|
||||
item = self.get_obj()
|
||||
item_content = item.get_content()
|
||||
obj = self.get_obj()
|
||||
content = obj.get_content()
|
||||
|
||||
# max execution time on regex
|
||||
res = self.regex_findall(self.onion_regex, item.get_id(), item_content)
|
||||
res = self.regex_findall(self.onion_regex, obj.get_id(), content)
|
||||
for x in res:
|
||||
# String to tuple
|
||||
x = x[2:-2].replace(" '", "").split("',")
|
||||
|
|
@ -92,16 +91,14 @@ class Onion(AbstractModule):
|
|||
if onion_urls:
|
||||
if crawlers.is_crawler_activated():
|
||||
for domain in domains:
|
||||
task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0,
|
||||
task_uuid = crawlers.create_task(domain, parent=obj.get_id(), priority=0,
|
||||
har=self.har, screenshot=self.screenshot)
|
||||
if task_uuid:
|
||||
print(f'{domain} added to crawler queue: {task_uuid}')
|
||||
else:
|
||||
to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
|
||||
self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
|
||||
print(f'Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
|
||||
|
||||
# TAG Item
|
||||
# TAG Object
|
||||
tag = 'infoleak:automatic-detection="onion"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ class Phone(AbstractModule):
|
|||
tag = 'infoleak:automatic-detection="phone-number"'
|
||||
self.add_message_to_queue(message=tag, queue='Tags')
|
||||
|
||||
self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
|
||||
self.logger.info(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
|
||||
|
||||
# # List of the regex results in the Item, may be null
|
||||
# results = self.REG_PHONE.findall(content)
|
||||
|
|
@ -70,7 +70,7 @@ class Phone(AbstractModule):
|
|||
# # If the list is greater than 4, we consider the Item may contain a list of phone numbers
|
||||
# if len(results) > 4:
|
||||
# self.logger.debug(results)
|
||||
# self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
|
||||
# self.logger.info(f'{item.get_id()} contains PID (phone numbers)')
|
||||
#
|
||||
# msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
|
||||
# self.add_message_to_queue(msg, 'Tags')
|
||||
|
|
@ -88,7 +88,7 @@ class Phone(AbstractModule):
|
|||
# pass
|
||||
# for country_code in stats:
|
||||
# if stats[country_code] > 4:
|
||||
# self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
|
||||
# self.logger.info(f'{item.get_id()} contains Phone numbers with country code {country_code}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ import sys
|
|||
import re
|
||||
import urllib.request
|
||||
|
||||
from datetime import datetime
|
||||
from pyfaup.faup import Faup
|
||||
# from pyfaup.faup import Faup
|
||||
from urllib.parse import unquote
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
|
|
@ -25,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
# from lib.ConfigLoader import ConfigLoader
|
||||
# from lib import Statistics
|
||||
|
||||
class SQLInjectionDetection(AbstractModule):
|
||||
|
|
@ -39,22 +37,19 @@ class SQLInjectionDetection(AbstractModule):
|
|||
def __init__(self):
|
||||
super(SQLInjectionDetection, self).__init__()
|
||||
|
||||
self.faup = Faup()
|
||||
# self.faup = Faup()
|
||||
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
url = message
|
||||
item = self.get_obj()
|
||||
|
||||
if self.is_sql_injection(url):
|
||||
self.faup.decode(url)
|
||||
url_parsed = self.faup.get()
|
||||
# self.faup.decode(url)
|
||||
# url_parsed = self.faup.get()
|
||||
|
||||
print(f"Detected SQL in URL: {item.id}")
|
||||
print(f"Detected SQL in URL: {self.obj.get_global_id()}")
|
||||
print(urllib.request.unquote(url))
|
||||
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
|
||||
self.redis_logger.warning(to_print)
|
||||
|
||||
# Tag
|
||||
tag = f'infoleak:automatic-detection="sql-injection"'
|
||||
|
|
|
|||
|
|
@ -167,11 +167,11 @@ class SentimentAnalysis(AbstractModule):
|
|||
provider_timestamp = provider + '_' + str(timestamp)
|
||||
self.db.incr('UniqID')
|
||||
UniqID = self.db.get('UniqID')
|
||||
self.redis_logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
|
||||
self.logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
|
||||
self.db.sadd(provider_timestamp, UniqID)
|
||||
self.db.set(UniqID, avg_score)
|
||||
else:
|
||||
self.redis_logger.debug(f'Dropped:{p_MimeType}')
|
||||
self.logger.debug(f'Dropped:{p_MimeType}')
|
||||
|
||||
|
||||
def isJSON(self, content):
|
||||
|
|
|
|||
|
|
@ -137,22 +137,21 @@ class SubmitPaste(AbstractModule):
|
|||
"""
|
||||
Create a paste for given file
|
||||
"""
|
||||
self.redis_logger.debug('manage')
|
||||
|
||||
if os.path.exists(file_full_path):
|
||||
self.redis_logger.debug(f'file exists {file_full_path}')
|
||||
self.logger.debug(f'file exists {file_full_path}')
|
||||
|
||||
file_size = os.stat(file_full_path).st_size
|
||||
self.redis_logger.debug(f'file size {file_size}')
|
||||
self.logger.debug(f'file size {file_size}')
|
||||
# Verify file length
|
||||
if file_size < SubmitPaste.FILE_MAX_SIZE:
|
||||
# TODO sanitize filename
|
||||
filename = file_full_path.split('/')[-1]
|
||||
self.redis_logger.debug(f'sanitize filename {filename}')
|
||||
self.redis_logger.debug('file size allowed')
|
||||
self.logger.debug(f'sanitize filename {filename}')
|
||||
self.logger.debug('file size allowed')
|
||||
|
||||
if not '.' in filename:
|
||||
self.redis_logger.debug('no extension for filename')
|
||||
self.logger.debug('no extension for filename')
|
||||
try:
|
||||
# Read file
|
||||
with open(file_full_path,'r') as f:
|
||||
|
|
@ -165,14 +164,14 @@ class SubmitPaste(AbstractModule):
|
|||
else:
|
||||
file_type = filename.rsplit('.', 1)[1]
|
||||
file_type = file_type.lower()
|
||||
self.redis_logger.debug(f'file ext {file_type}')
|
||||
self.logger.debug(f'file ext {file_type}')
|
||||
|
||||
if file_type in SubmitPaste.ALLOWED_EXTENSIONS:
|
||||
self.redis_logger.debug('Extension allowed')
|
||||
self.logger.debug('Extension allowed')
|
||||
# TODO enum of possible file extension ?
|
||||
# TODO verify file hash with virus total ?
|
||||
if not self._is_compressed_type(file_type):
|
||||
self.redis_logger.debug('Plain text file')
|
||||
self.logger.debug('Plain text file')
|
||||
# plain txt file
|
||||
with open(file_full_path,'r') as f:
|
||||
content = f.read()
|
||||
|
|
@ -197,7 +196,7 @@ class SubmitPaste(AbstractModule):
|
|||
# except:
|
||||
# self.abord_file_submission(uuid, "file decompression error")
|
||||
# raise
|
||||
# self.redis_logger.debug('unpacking {} file'.format(files.unpacker))
|
||||
# self.logger.debug('unpacking {} file'.format(files.unpacker))
|
||||
# if(not files.children):
|
||||
# self.abord_file_submission(uuid, "Empty compressed file")
|
||||
# raise
|
||||
|
|
@ -209,11 +208,11 @@ class SubmitPaste(AbstractModule):
|
|||
# self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source)
|
||||
# n = n + 1
|
||||
# else:
|
||||
# self.redis_logger.error("Error in module %s: bad extention"%(self.module_name))
|
||||
# self.logger.error("Error in module %s: bad extention"%(self.module_name))
|
||||
# self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
|
||||
|
||||
# except FileNotFoundError:
|
||||
# self.redis_logger.error("Error in module %s: file not found"%(self.module_name))
|
||||
# self.logger.error("Error in module %s: file not found"%(self.module_name))
|
||||
# self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
|
||||
|
||||
else:
|
||||
|
|
@ -248,7 +247,7 @@ class SubmitPaste(AbstractModule):
|
|||
|
||||
# delete uuid
|
||||
self.r_serv_db.srem('submitted:uuid', uuid)
|
||||
self.redis_logger.debug(f'{uuid} all file submitted')
|
||||
self.logger.debug(f'{uuid} all file submitted')
|
||||
print(f'{uuid} all file submitted')
|
||||
|
||||
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
|
||||
|
|
@ -262,11 +261,11 @@ class SubmitPaste(AbstractModule):
|
|||
|
||||
full_path = os.path.join(ITEMS_FOLDER, save_path)
|
||||
|
||||
self.redis_logger.debug(f'file path of the paste {full_path}')
|
||||
self.logger.debug(f'file path of the paste {full_path}')
|
||||
|
||||
if not os.path.isfile(full_path):
|
||||
# file not exists in AIL paste directory
|
||||
self.redis_logger.debug(f"new paste {paste_content}")
|
||||
self.logger.debug(f"new paste {paste_content}")
|
||||
|
||||
gzip64encoded = self._compress_encode_content(paste_content, uuid)
|
||||
|
||||
|
|
@ -274,7 +273,7 @@ class SubmitPaste(AbstractModule):
|
|||
|
||||
# use relative path
|
||||
rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1)
|
||||
self.redis_logger.debug(f"relative path {rel_item_path}")
|
||||
self.logger.debug(f"relative path {rel_item_path}")
|
||||
|
||||
item = Item(rel_item_path)
|
||||
|
||||
|
|
@ -295,12 +294,12 @@ class SubmitPaste(AbstractModule):
|
|||
if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'):
|
||||
self.r_serv_log_submit.set(f'{uuid}:end', 1)
|
||||
|
||||
self.redis_logger.debug(f' {rel_item_path} send to Mixer')
|
||||
self.logger.debug(f' {rel_item_path} send to Mixer')
|
||||
print(f' {rel_item_path} send to Mixer')
|
||||
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
self.redis_logger.debug("paste submitted")
|
||||
self.logger.debug("paste submitted")
|
||||
else:
|
||||
self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
|
||||
|
||||
|
|
@ -316,7 +315,7 @@ class SubmitPaste(AbstractModule):
|
|||
return gzip64encoded
|
||||
|
||||
def addError(self, uuid, errorMessage):
|
||||
self.redis_logger.debug(errorMessage)
|
||||
self.logger.debug(errorMessage)
|
||||
print(errorMessage)
|
||||
error = self.r_serv_log_submit.get(f'{uuid}:error')
|
||||
if error is not None:
|
||||
|
|
@ -324,7 +323,7 @@ class SubmitPaste(AbstractModule):
|
|||
self.r_serv_log_submit.incr(f'{uuid}:nb_end')
|
||||
|
||||
def abord_file_submission(self, uuid, errorMessage):
|
||||
self.redis_logger.debug(f'abord {uuid}, {errorMessage}')
|
||||
self.logger.debug(f'abord {uuid}, {errorMessage}')
|
||||
|
||||
self.addError(uuid, errorMessage)
|
||||
self.r_serv_log_submit.set(f'{uuid}:end', 1)
|
||||
|
|
|
|||
|
|
@ -85,8 +85,6 @@ class Urls(AbstractModule):
|
|||
if len(l_urls) > 0:
|
||||
to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
print(to_print)
|
||||
# .debug ???
|
||||
# self.redis_logger.info(f'{to_print}Detected {len(l_urls)} URL;{item.get_id()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from pubsublogger import publisher
|
||||
from lib import ail_logger
|
||||
from lib.ail_queues import AILQueue
|
||||
from lib import regex_helper
|
||||
|
|
@ -51,15 +50,6 @@ class AbstractModule(ABC):
|
|||
self.obj = None
|
||||
self.sha256_mess = None
|
||||
|
||||
# Init Redis Logger
|
||||
self.redis_logger = publisher
|
||||
# Port of the redis instance used by pubsublogger
|
||||
self.redis_logger.port = 6380
|
||||
# Channel name to publish logs
|
||||
# # TODO: refactor logging
|
||||
# If provided could be a namespaced channel like script:<ModuleName>
|
||||
self.redis_logger.channel = 'Script'
|
||||
|
||||
# Cache key
|
||||
self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
|
||||
self.max_execution_time = 30
|
||||
|
|
|
|||
|
|
@ -42,12 +42,11 @@ class Retro_Hunt_Module(AbstractModule):
|
|||
self.obj = None
|
||||
self.tags = []
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
# # TODO: # start_time
|
||||
# # end_time
|
||||
def compute(self, task_uuid):
|
||||
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
|
||||
print(f'starting Retro hunt task {task_uuid}')
|
||||
self.progress = 0
|
||||
# First launch
|
||||
|
|
@ -58,7 +57,7 @@ class Retro_Hunt_Module(AbstractModule):
|
|||
timeout = self.retro_hunt.get_timeout()
|
||||
self.tags = self.retro_hunt.get_tags()
|
||||
|
||||
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
|
||||
self.logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
|
||||
|
||||
# Filters
|
||||
filters = self.retro_hunt.get_filters()
|
||||
|
|
@ -111,7 +110,6 @@ class Retro_Hunt_Module(AbstractModule):
|
|||
# Completed
|
||||
self.retro_hunt.complete()
|
||||
print(f'Retro Hunt {task_uuid} completed')
|
||||
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
|
||||
|
||||
def update_progress(self):
|
||||
if self.nb_objs == 0:
|
||||
|
|
@ -128,7 +126,6 @@ class Retro_Hunt_Module(AbstractModule):
|
|||
# print(data)
|
||||
task_uuid = data['namespace']
|
||||
|
||||
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
|
||||
print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
|
||||
|
||||
self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id)
|
||||
|
|
@ -158,16 +155,15 @@ class Retro_Hunt_Module(AbstractModule):
|
|||
task_uuid = Tracker.get_retro_hunt_task_to_start()
|
||||
if task_uuid:
|
||||
# Module processing with the message from the queue
|
||||
self.redis_logger.debug(task_uuid)
|
||||
self.logger.debug(task_uuid)
|
||||
# try:
|
||||
self.compute(task_uuid)
|
||||
# except Exception as err:
|
||||
# self.redis_logger.error(f'Error in module {self.module_name}: {err}')
|
||||
# self.logger.error(f'Error in module {self.module_name}: {err}')
|
||||
# # Remove uuid ref
|
||||
# self.remove_submit_uuid(uuid)
|
||||
else:
|
||||
# Wait before next process
|
||||
self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s')
|
||||
time.sleep(self.pending_seconds)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -47,14 +47,13 @@ class Tracker_Regex(AbstractModule):
|
|||
self.exporters = {'mail': MailExporterTracker(),
|
||||
'webhook': WebHookExporterTracker()}
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh Tracked regex
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
||||
self.tracked_regexs = Tracker.get_tracked_regexs()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked regex refreshed')
|
||||
print('Tracked regex refreshed')
|
||||
|
||||
obj = self.get_obj()
|
||||
|
|
@ -117,7 +116,6 @@ class Tracker_Regex(AbstractModule):
|
|||
continue
|
||||
|
||||
print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
|
||||
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
|
||||
|
||||
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -62,20 +62,18 @@ class Tracker_Term(AbstractModule):
|
|||
self.exporters = {'mail': MailExporterTracker(),
|
||||
'webhook': WebHookExporterTracker()}
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh Tracked term
|
||||
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
|
||||
self.tracked_words = Tracker.get_tracked_words()
|
||||
self.last_refresh_word = time.time()
|
||||
self.redis_logger.debug('Tracked word refreshed')
|
||||
print('Tracked word refreshed')
|
||||
|
||||
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
|
||||
self.tracked_sets = Tracker.get_tracked_sets()
|
||||
self.last_refresh_set = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
obj = self.get_obj()
|
||||
|
|
@ -93,7 +91,7 @@ class Tracker_Term(AbstractModule):
|
|||
try:
|
||||
dict_words_freq = Tracker.get_text_word_frequency(content)
|
||||
except TimeoutException:
|
||||
self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout")
|
||||
self.logger.warning(f"{self.obj.get_global_id()} processing timeout")
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
|
|
@ -125,7 +123,6 @@ class Tracker_Term(AbstractModule):
|
|||
continue
|
||||
|
||||
print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}')
|
||||
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}')
|
||||
|
||||
tracker.add(obj.get_type(), obj.get_subtype(), obj_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -43,14 +43,13 @@ class Tracker_Typo_Squatting(AbstractModule):
|
|||
self.exporters = {'mail': MailExporterTracker(),
|
||||
'webhook': WebHookExporterTracker()}
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh Tracked typo
|
||||
if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
|
||||
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
|
||||
self.last_refresh_typosquatting = time.time()
|
||||
self.redis_logger.debug('Tracked typosquatting refreshed')
|
||||
print('Tracked typosquatting refreshed')
|
||||
|
||||
host = message
|
||||
|
|
@ -76,7 +75,6 @@ class Tracker_Typo_Squatting(AbstractModule):
|
|||
continue
|
||||
|
||||
print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}')
|
||||
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}')
|
||||
|
||||
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -44,14 +44,13 @@ class Tracker_Yara(AbstractModule):
|
|||
self.exporters = {'mail': MailExporterTracker(),
|
||||
'webhook': WebHookExporterTracker()}
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh YARA list
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
self.rules = Tracker.get_tracked_yara_rules()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
self.obj = self.get_obj()
|
||||
|
|
@ -69,11 +68,9 @@ class Tracker_Yara(AbstractModule):
|
|||
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}')
|
||||
print(f'{self.obj.get_global_id()}: {yara_match}')
|
||||
except yara.TimeoutError:
|
||||
print(f'{self.obj.get_id()}: yara scanning timed out')
|
||||
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
|
||||
|
||||
def convert_byte_offset_to_string(self, b_content, offset):
|
||||
byte_chunk = b_content[:offset + 1]
|
||||
|
|
|
|||
|
|
@ -25,9 +25,6 @@ xxhash>3.1.0
|
|||
zmq
|
||||
pyzmq>19.0.0
|
||||
|
||||
# Logging
|
||||
logbook
|
||||
pubsublogger
|
||||
|
||||
# Tokeniser
|
||||
nltk>3.4.5
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import sys
|
|||
##################################
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
from pubsublogger import publisher
|
||||
|
||||
# FLASK #
|
||||
app = None
|
||||
|
|
@ -34,13 +33,6 @@ r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
|
|||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint
|
||||
r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint
|
||||
|
||||
# Logger (Redis)
|
||||
redis_logger = publisher
|
||||
# Port of the redis instance used by pubsublogger
|
||||
redis_logger.port = 6380
|
||||
# Channel name to publish logs
|
||||
redis_logger.channel = 'Flask'
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
|
||||
#### VARIABLES ####
|
||||
|
|
|
|||
|
|
@ -39,8 +39,6 @@ baseUrl = Flask_config.baseUrl
|
|||
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
|
||||
|
||||
logger = Flask_config.redis_logger
|
||||
|
||||
|
||||
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
|
||||
|
||||
|
|
@ -57,11 +55,9 @@ def limit_content_length():
|
|||
def decorator(f):
|
||||
@wraps(f)
|
||||
def wrapper(*args, **kwargs):
|
||||
logger.debug('decorator')
|
||||
cl = request.content_length
|
||||
if cl is not None:
|
||||
if cl > Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE or ('file' not in request.files and cl > Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE):
|
||||
logger.debug('abort')
|
||||
abort(413)
|
||||
return f(*args, **kwargs)
|
||||
return wrapper
|
||||
|
|
@ -75,7 +71,6 @@ def allowed_file(filename):
|
|||
return True
|
||||
else:
|
||||
file_ext = filename.rsplit('.', 1)[1].lower()
|
||||
logger.debug(file_ext)
|
||||
return file_ext in Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS
|
||||
|
||||
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
|
||||
|
|
@ -111,7 +106,6 @@ def PasteSubmit_page():
|
|||
@login_user_no_api
|
||||
@limit_content_length()
|
||||
def submit():
|
||||
logger.debug('submit')
|
||||
|
||||
password = request.form['archive_pass']
|
||||
ltags = request.form['tags_taxonomies']
|
||||
|
|
@ -124,12 +118,10 @@ def submit():
|
|||
paste_source = paste_source.replace('/', '')[:80]
|
||||
if paste_source in ['crawled', 'tests']:
|
||||
content = 'Invalid source'
|
||||
logger.info(paste_source)
|
||||
return content, 400
|
||||
|
||||
if not re.match('^[0-9a-zA-Z-_\+@#&\.;=:!]*$', paste_source):
|
||||
content = f'Invalid source name: Forbidden character(s)'
|
||||
logger.info(content)
|
||||
return content, 400
|
||||
|
||||
is_file = False
|
||||
|
|
@ -139,8 +131,6 @@ def submit():
|
|||
if file_import.filename:
|
||||
is_file = True
|
||||
|
||||
logger.debug(f'is file ? {is_file}')
|
||||
|
||||
submitted_tag = 'infoleak:submission="manual"'
|
||||
|
||||
# active taxonomies
|
||||
|
|
@ -149,13 +139,11 @@ def submit():
|
|||
active_galaxies = Tag.get_active_galaxies()
|
||||
|
||||
if ltags or ltagsgalaxies:
|
||||
logger.debug(f'ltags ? {ltags} {ltagsgalaxies}')
|
||||
ltags = Tag.unpack_str_tags_list(ltags)
|
||||
ltagsgalaxies = Tag.unpack_str_tags_list(ltagsgalaxies)
|
||||
|
||||
if not Tag.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
|
||||
content = 'INVALID TAGS'
|
||||
logger.info(content)
|
||||
return content, 400
|
||||
|
||||
# add submitted tags
|
||||
|
|
@ -164,36 +152,28 @@ def submit():
|
|||
ltags.append(submitted_tag)
|
||||
|
||||
if is_file:
|
||||
logger.debug('file management')
|
||||
|
||||
if allowed_file(file_import.filename):
|
||||
logger.debug('file extension allowed')
|
||||
|
||||
# get UUID
|
||||
UUID = str(uuid.uuid4())
|
||||
|
||||
# create submitted dir
|
||||
if not os.path.exists(UPLOAD_FOLDER):
|
||||
logger.debug('create folder')
|
||||
os.makedirs(UPLOAD_FOLDER)
|
||||
|
||||
if '.' not in file_import.filename:
|
||||
logger.debug('add UUID to path')
|
||||
full_path = os.path.join(UPLOAD_FOLDER, UUID)
|
||||
else:
|
||||
if file_import.filename[-6:] == 'tar.gz':
|
||||
logger.debug('file extension is tar.gz')
|
||||
file_type = 'tar.gz'
|
||||
else:
|
||||
file_type = file_import.filename.rsplit('.', 1)[1]
|
||||
logger.debug(f'file type {file_type}')
|
||||
name = UUID + '.' + file_type
|
||||
full_path = os.path.join(UPLOAD_FOLDER, name)
|
||||
logger.debug(f'full path {full_path}')
|
||||
|
||||
# Flask verify the file size
|
||||
file_import.save(full_path)
|
||||
logger.debug('file saved')
|
||||
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
|
||||
|
||||
|
|
@ -204,18 +184,13 @@ def submit():
|
|||
|
||||
else:
|
||||
content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension'
|
||||
logger.info(content)
|
||||
return content, 400
|
||||
|
||||
elif paste_content != '':
|
||||
logger.debug(f'entering text paste management')
|
||||
if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE:
|
||||
logger.debug(f'size {sys.getsizeof(paste_content)}')
|
||||
# get id
|
||||
UUID = str(uuid.uuid4())
|
||||
logger.debug('create import')
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, source=paste_source)
|
||||
logger.debug('import OK')
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
active_galaxies = active_galaxies,
|
||||
|
|
@ -223,11 +198,9 @@ def submit():
|
|||
|
||||
else:
|
||||
content = f'text paste size is over {Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE} bytes limit'
|
||||
logger.info(content)
|
||||
return content, 400
|
||||
|
||||
content = 'submit aborded'
|
||||
logger.error(content)
|
||||
return content, 400
|
||||
|
||||
return PasteSubmit_page()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue