AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 08:26:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chg: [pubsublogger] remove old redis_logger

Browse source
This commit is contained in:
terrtia 2024-12-09 13:54:54 +01:00
parent feabe9f5df
commit 849977f0c2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
32 changed files with 112 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
bin/LAUNCH.sh
View file

@ -615,14 +615,13 @@ function launch_all {
update; update;
launch_redis; launch_redis;
launch_kvrocks; launch_kvrocks;
launch_logs;
launch_scripts; launch_scripts;
launch_flask; launch_flask;
} }
function menu_display { function menu_display {
options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
menu() { menu() {
echo "What do you want to Launch?:" echo "What do you want to Launch?:"
@ -653,9 +652,6 @@ function menu_display {
Kvrocks) Kvrocks)
launch_kvrocks; launch_kvrocks;
;; ;;
Logs)
launch_logs;
;;
Scripts) Scripts)
launch_scripts; launch_scripts;
;; ;;

1
bin/core/Sync_module.py
View file

@ -88,7 +88,6 @@ class Sync_module(AbstractModule):
if self.last_refresh_queues < time.time(): if self.last_refresh_queues < time.time():
timeout_processed_objs() timeout_processed_objs()
self.last_refresh_queues = time.time() + 120 self.last_refresh_queues = time.time() + 120
self.redis_logger.debug('Timeout queues')
# print('Timeout queues') # print('Timeout queues')
# Get one message (paste) from the QueueIn (copy of Redis_Global publish) # Get one message (paste) from the QueueIn (copy of Redis_Global publish)

16
bin/modules/ApiKey.py
View file

@ -20,7 +20,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class ApiKey(AbstractModule): class ApiKey(AbstractModule):
"""ApiKey module for AIL framework""" """ApiKey module for AIL framework"""
@ -48,20 +47,19 @@ class ApiKey(AbstractModule):
def compute(self, message, r_result=False): def compute(self, message, r_result=False):
score = message score = message
item = self.get_obj() obj = self.get_obj()
item_content = item.get_content() content = obj.get_content()
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True) google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True) aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True)
if aws_access_key: if aws_access_key:
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True) aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True)
if aws_access_key or google_api_key: if aws_access_key or google_api_key:
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = obj.get_global_id()
if google_api_key: if google_api_key:
print(f'found google api key: {to_print}') print(f'found google api key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="google-api-key"' tag = 'infoleak:automatic-detection="google-api-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
@ -69,10 +67,8 @@ class ApiKey(AbstractModule):
# # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
if aws_access_key: if aws_access_key:
print(f'found AWS key: {to_print}') print(f'found AWS key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
if aws_secret_key: if aws_secret_key:
print(f'found AWS secret key') print(f'found AWS secret key')
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="aws-key"' tag = 'infoleak:automatic-detection="aws-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

5
bin/modules/Categ.py
View file

@ -65,7 +65,7 @@ class Categ(AbstractModule):
self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold") self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
self.reload_categ_words() self.reload_categ_words()
self.redis_logger.info("Script Categ started") self.logger.info("Script Categ started")
# # TODO: trigger reload on change ( save last reload time, ...) # # TODO: trigger reload on change ( save last reload time, ...)
def reload_categ_words(self): def reload_categ_words(self):
@ -103,9 +103,6 @@ class Categ(AbstractModule):
print(msg, categ) print(msg, categ)
self.add_message_to_queue(message=msg, queue=categ) self.add_message_to_queue(message=msg, queue=categ)
self.redis_logger.debug(
f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
if r_result: if r_result:
return categ_found return categ_found

22
bin/modules/Credential.py
View file

@ -29,7 +29,6 @@ Redis organization:
import os import os
import sys import sys
import time import time
from datetime import datetime
from pyfaup.faup import Faup from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -37,9 +36,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib import ConfigLoader from lib import ConfigLoader
# from lib import Statistics
class Credential(AbstractModule): class Credential(AbstractModule):
@ -80,40 +77,36 @@ class Credential(AbstractModule):
self.pending_seconds = 10 self.pending_seconds = 10
# Send module state to logs # Send module state to logs
self.redis_logger.info(f"Module {self.module_name} initialized") self.logger.info(f"Module {self.module_name} initialized")
def compute(self, message): def compute(self, message):
count = message obj = self.get_obj()
item = self.get_obj()
item_content = item.get_content() content = obj.get_content()
# TODO: USE SETS # TODO: USE SETS
# Extract all credentials # Extract all credentials
all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content) all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content)
if all_credentials: if all_credentials:
nb_cred = len(all_credentials) nb_cred = len(all_credentials)
message = f'Checked {nb_cred} credentials found.' message = f'Checked {nb_cred} credentials found.'
all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True) all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True)
if all_sites: if all_sites:
discovered_sites = ', '.join(all_sites) discovered_sites = ', '.join(all_sites)
message += f' Related websites: {discovered_sites}' message += f' Related websites: {discovered_sites}'
print(message) print(message)
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
# num of creds above threshold, publish an alert # num of creds above threshold, publish an alert
if nb_cred > self.criticalNumberToAlert: if nb_cred > self.criticalNumberToAlert:
print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}") print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
self.redis_logger.warning(to_print)
tag = 'infoleak:automatic-detection="credential"' tag = 'infoleak:automatic-detection="credential"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content) site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content)
creds_sites = {} creds_sites = {}
@ -162,8 +155,7 @@ class Credential(AbstractModule):
# for tld in nb_tlds: # for tld in nb_tlds:
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld]) # Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
else: else:
self.redis_logger.info(to_print) print(f'found {nb_cred} credentials {self.obj.get_global_id()}')
print(f'found {nb_cred} credentials')
# # TODO: # FIXME: TEMP DESABLE # # TODO: # FIXME: TEMP DESABLE
# # For searching credential in termFreq # # For searching credential in termFreq

17
bin/modules/CreditCards.py
View file

@ -7,7 +7,7 @@ The CreditCards Module
This module is consuming the Redis-list created by the Categ module. This module is consuming the Redis-list created by the Categ module.
It apply credit card regexes on item content and warn if a valid card number is found. It apply credit card regexes on object content and warn if a valid card number is found.
""" """
@ -23,7 +23,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from packages import lib_refine from packages import lib_refine
class CreditCards(AbstractModule): class CreditCards(AbstractModule):
@ -68,10 +67,9 @@ class CreditCards(AbstractModule):
return extracted return extracted
def compute(self, message, r_result=False): def compute(self, message, r_result=False):
score = message obj = self.get_obj()
item = self.get_obj() content = obj.get_content()
content = item.get_content() all_cards = self.regex_findall(self.regex, obj.id, content)
all_cards = self.regex_findall(self.regex, item.id, content)
if len(all_cards) > 0: if len(all_cards) > 0:
# self.logger.debug(f'All matching {all_cards}') # self.logger.debug(f'All matching {all_cards}')
@ -84,11 +82,8 @@ class CreditCards(AbstractModule):
creditcard_set.add(valid_card) creditcard_set.add(valid_card)
# print(creditcard_set) # print(creditcard_set)
to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
if creditcard_set: if creditcard_set:
mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}' print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}')
print(mess)
self.redis_logger.warning(mess)
tag = 'infoleak:automatic-detection="credit-card"' tag = 'infoleak:automatic-detection="credit-card"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
@ -96,7 +91,7 @@ class CreditCards(AbstractModule):
if r_result: if r_result:
return creditcard_set return creditcard_set
else: else:
self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}') print(f'CreditCard related;{self.obj.get_global_id()}')
if __name__ == '__main__': if __name__ == '__main__':

10
bin/modules/Cryptocurrencies.py
View file

@ -28,7 +28,6 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.CryptoCurrencies import CryptoCurrency from lib.objects.CryptoCurrencies import CryptoCurrency
from lib.objects.Items import Item
################################## ##################################
################################## ##################################
@ -151,17 +150,10 @@ class Cryptocurrencies(AbstractModule, ABC):
# debug # debug
print(private_keys) print(private_keys)
to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(),
item.get_date(),
item.get_basename())
self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
else: else:
private_keys = [] private_keys = []
to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys" print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}")
print(to_print)
self.redis_logger.warning(to_print)
if __name__ == '__main__': if __name__ == '__main__':

5
bin/modules/CveModule.py
View file

@ -44,7 +44,6 @@ class CveModule(AbstractModule):
self.logger.info(f'Module {self.module_name} initialized') self.logger.info(f'Module {self.module_name} initialized')
def compute(self, message): def compute(self, message):
count = message
item = self.get_obj() item = self.get_obj()
item_id = item.get_id() item_id = item.get_id()
@ -56,9 +55,7 @@ class CveModule(AbstractModule):
cve = Cves.Cve(cve_id) cve = Cves.Cve(cve_id)
cve.add(date, item) cve.add(date, item)
warning = f'{self.obj.get_global_id()} contains CVEs {cves}' print(f'{self.obj.get_global_id()} contains CVEs {cves}')
print(warning)
self.redis_logger.warning(warning)
tag = 'infoleak:automatic-detection="cve"' tag = 'infoleak:automatic-detection="cve"'
# Send to Tags Queue # Send to Tags Queue

10
bin/modules/DomClassifier.py
View file

@ -58,10 +58,6 @@ class DomClassifier(AbstractModule):
def compute(self, message, r_result=False): def compute(self, message, r_result=False):
host = message host = message
item = self.get_obj()
item_basename = item.get_basename()
item_date = item.get_date()
item_source = item.get_source()
try: try:
self.dom_classifier.text(rawtext=host) self.dom_classifier.text(rawtext=host)
@ -82,19 +78,19 @@ class DomClassifier(AbstractModule):
localizeddomains = self.dom_classifier.include(expression=self.cc_tld) localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}") self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
if self.cc: if self.cc:
localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}") self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
if r_result: if r_result:
return self.dom_classifier.vdomain return self.dom_classifier.vdomain
except IOError as err: except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed")
raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}") raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")

5
bin/modules/Duplicates.py
View file

@ -27,7 +27,6 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib import Duplicate from lib import Duplicate
from lib.objects.Items import Item
class Duplicates(AbstractModule): class Duplicates(AbstractModule):
@ -92,11 +91,11 @@ class Duplicates(AbstractModule):
Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id()) Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
if nb_duplicates: if nb_duplicates:
self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}') self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}')
y = time.time() y = time.time()
print(f'{self.obj.get_global_id()} Processed in {y-x} sec') print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
# self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x)) # self.logger.debug('{}Processed in {} sec'.format(to_print, y-x))
if __name__ == "__main__": if __name__ == "__main__":

1
bin/modules/Global.py
View file

@ -74,7 +74,6 @@ class Global(AbstractModule):
if int(difftime) > 30: if int(difftime) > 30:
to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s' to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s'
print(to_print) print(to_print)
self.redis_logger.debug(to_print)
self.time_last_stats = time.time() self.time_last_stats = time.time()
self.processed_item = 0 self.processed_item = 0

9
bin/modules/IPAddress.py
View file

@ -25,7 +25,6 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import regex_helper from lib import regex_helper
# TODO REWRITE ME -> PERF + IPV6 + Tracker ? # TODO REWRITE ME -> PERF + IPV6 + Tracker ?
@ -66,11 +65,11 @@ class IPAddress(AbstractModule):
if not self.ip_networks: if not self.ip_networks:
return None return None
item = self.get_obj() obj = self.get_obj()
content = item.get_content() content = obj.get_content()
# list of the regex results in the Item # list of the regex results in the Item
results = self.regex_findall(self.re_ipv4, item.get_id(), content) results = self.regex_findall(self.re_ipv4, obj.get_id(), content)
results = set(results) results = set(results)
matching_ips = [] matching_ips = []
for ip in results: for ip in results:
@ -83,7 +82,6 @@ class IPAddress(AbstractModule):
if len(matching_ips) > 0: if len(matching_ips) > 0:
self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs') self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
# Tag message with IP # Tag message with IP
tag = 'infoleak:automatic-detection="ip"' tag = 'infoleak:automatic-detection="ip"'
@ -93,4 +91,3 @@ class IPAddress(AbstractModule):
if __name__ == "__main__": if __name__ == "__main__":
module = IPAddress() module = IPAddress()
module.run() module.run()
# module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')

21
bin/modules/Iban.py
View file

@ -24,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item # from lib.ConfigLoader import ConfigLoader
from lib.ConfigLoader import ConfigLoader
# from lib import Statistics # from lib import Statistics
class Iban(AbstractModule): class Iban(AbstractModule):
@ -40,7 +39,7 @@ class Iban(AbstractModule):
def __init__(self, queue=True): def __init__(self, queue=True):
super(Iban, self).__init__(queue=queue) super(Iban, self).__init__(queue=queue)
# Waiting time in secondes between to message proccessed # Waiting time in seconds between to message processed
self.pending_seconds = 10 self.pending_seconds = 10
self.regex_timeout = 30 self.regex_timeout = 30
@ -49,7 +48,7 @@ class Iban(AbstractModule):
self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$') self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
# Send module state to logs # Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized') self.logger.info(f'Module {self.module_name} initialized')
def get_iban_number(self, iban): def get_iban_number(self, iban):
return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN) return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN)
@ -73,29 +72,27 @@ class Iban(AbstractModule):
return extracted return extracted
def compute(self, message): def compute(self, message):
item = self.get_obj() obj = self.get_obj()
item_id = item.get_id() obj_id = obj.get_id()
ibans = self.regex_findall(self.iban_regex, item_id, item.get_content()) ibans = self.regex_findall(self.iban_regex, obj_id, obj.get_content())
if ibans: if ibans:
valid_ibans = set() valid_ibans = set()
for iban in ibans: for iban in ibans:
iban = iban[1:-1].replace("'", "").split(',') iban = iban[1:-1].replace("'", "").split(',')
iban = iban[0]+iban[1]+iban[2] iban = iban[0]+iban[1]+iban[2]
iban = ''.join(e for e in iban if e.isalnum()) iban = ''.join(e for e in iban if e.isalnum())
if self.regex_findall(self.iban_regex_verify, item_id, iban): if self.regex_findall(self.iban_regex_verify, obj_id, iban):
print(f'checking {iban}') print(f'checking {iban}')
if self.is_valid_iban(iban): if self.is_valid_iban(iban):
valid_ibans.add(iban) valid_ibans.add(iban)
if valid_ibans: if valid_ibans:
print(f'{valid_ibans} ibans {item_id}') print(f'{valid_ibans} ibans {self.obj.get_global_id()}')
date = datetime.datetime.now().strftime("%Y%m") # date = datetime.datetime.now().strftime("%Y%m")
# for iban in valid_ibans: # for iban in valid_ibans:
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1) # Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}')
# Tags # Tags
tag = 'infoleak:automatic-detection="iban"' tag = 'infoleak:automatic-detection="iban"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

29
bin/modules/Keys.py
View file

@ -24,7 +24,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class KeyEnum(Enum): class KeyEnum(Enum):
@ -56,14 +55,14 @@ class Keys(AbstractModule):
self.pending_seconds = 1 self.pending_seconds = 1
def compute(self, message): def compute(self, message):
item = self.get_obj() obj = self.get_obj()
content = item.get_content() content = obj.get_content()
# find = False # find = False
get_pgp_content = False get_pgp_content = False
if KeyEnum.PGP_MESSAGE.value in content: if KeyEnum.PGP_MESSAGE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message') print(f'{self.obj.get_global_id()} has a PGP enc message')
tag = 'infoleak:automatic-detection="pgp-message"' tag = 'infoleak:automatic-detection="pgp-message"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
@ -81,21 +80,21 @@ class Keys(AbstractModule):
get_pgp_content = True get_pgp_content = True
if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message') print(f'{self.obj.get_global_id()} has a pgp private key block message')
tag = 'infoleak:automatic-detection="pgp-private-key"' tag = 'infoleak:automatic-detection="pgp-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
get_pgp_content = True get_pgp_content = True
if KeyEnum.CERTIFICATE.value in content: if KeyEnum.CERTIFICATE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message') print(f'{self.obj.get_global_id()} has a certificate message')
tag = 'infoleak:automatic-detection="certificate"' tag = 'infoleak:automatic-detection="certificate"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.RSA_PRIVATE_KEY.value in content: if KeyEnum.RSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message') print(f'{self.obj.get_global_id()} has a RSA private key message')
print('rsa private key message found') print('rsa private key message found')
tag = 'infoleak:automatic-detection="rsa-private-key"' tag = 'infoleak:automatic-detection="rsa-private-key"'
@ -103,7 +102,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.PRIVATE_KEY.value in content: if KeyEnum.PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message') print(f'{self.obj.get_global_id()} has a private key message')
print('private key message found') print('private key message found')
tag = 'infoleak:automatic-detection="private-key"' tag = 'infoleak:automatic-detection="private-key"'
@ -111,7 +110,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message') print(f'{self.obj.get_global_id()} has an encrypted private key message')
print('encrypted private key message found') print('encrypted private key message found')
tag = 'infoleak:automatic-detection="encrypted-private-key"' tag = 'infoleak:automatic-detection="encrypted-private-key"'
@ -119,7 +118,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print(f'{self.obj.get_global_id()} has an openssh private key message')
print('openssh private key message found') print('openssh private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"' tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -127,7 +126,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message') print(f'{self.obj.get_global_id()} has an ssh2 private key message')
print('SSH2 private key message found') print('SSH2 private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"' tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -135,7 +134,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print(f'{self.obj.get_global_id()} has an openssh private key message')
print('OpenVPN Static key message found') print('OpenVPN Static key message found')
tag = 'infoleak:automatic-detection="vpn-static-key"' tag = 'infoleak:automatic-detection="vpn-static-key"'
@ -143,21 +142,21 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.DSA_PRIVATE_KEY.value in content: if KeyEnum.DSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message') print(f'{self.obj.get_global_id()} has a dsa private key message')
tag = 'infoleak:automatic-detection="dsa-private-key"' tag = 'infoleak:automatic-detection="dsa-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.EC_PRIVATE_KEY.value in content: if KeyEnum.EC_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message') print(f'{self.obj.get_global_id()} has an ec private key message')
tag = 'infoleak:automatic-detection="ec-private-key"' tag = 'infoleak:automatic-detection="ec-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.PUBLIC_KEY.value in content: if KeyEnum.PUBLIC_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message') print(f'{self.obj.get_global_id()} has a public key message')
tag = 'infoleak:automatic-detection="public-key"' tag = 'infoleak:automatic-detection="public-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

10
bin/modules/LibInjection.py
View file

@ -34,10 +34,9 @@ class LibInjection(AbstractModule):
self.faup = Faup() self.faup = Faup()
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
item = self.get_obj()
url = message url = message
self.faup.decode(url) self.faup.decode(url)
@ -66,12 +65,9 @@ class LibInjection(AbstractModule):
# print(f'query is sqli : {result_query}') # print(f'query is sqli : {result_query}')
if result_path['sqli'] is True or result_query['sqli'] is True: if result_path['sqli'] is True or result_query['sqli'] is True:
item_id = item.get_id()
print(f"Detected (libinjection) SQL in URL: {item_id}")
print(unquote(url))
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' self.logger.info(f'Detected SQL in URL;{self.obj.get_global_id()}')
self.redis_logger.warning(to_print) print(unquote(url))
# Add tag # Add tag
tag = 'infoleak:automatic-detection="sql-injection"' tag = 'infoleak:automatic-detection="sql-injection"'

3
bin/modules/MISP_Thehive_Auto_Push.py
View file

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.exceptions import MISPConnectionError from lib.exceptions import MISPConnectionError
from lib.objects.Items import Item
from lib import Tag from lib import Tag
from exporter.MISPExporter import MISPExporterAutoDaily from exporter.MISPExporter import MISPExporterAutoDaily
from exporter.TheHiveExporter import TheHiveExporterAlertTag from exporter.TheHiveExporter import TheHiveExporterAlertTag
@ -43,7 +42,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0: if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0:
self.tags = Tag.refresh_auto_push() self.tags = Tag.refresh_auto_push()
self.last_refresh = time.time() self.last_refresh = time.time()
self.redis_logger.info('Tags Auto Push refreshed') self.logger.debug('Tags Auto Push refreshed')
tag = message tag = message
item = self.get_obj() item = self.get_obj()

18
bin/modules/Mail.py
View file

@ -95,20 +95,20 @@ class Mail(AbstractModule):
# print() # print()
except dns.resolver.NoNameservers: except dns.resolver.NoNameservers:
self.redis_logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.') self.logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
print('NoNameserver, No non-broken nameservers are available to answer the query.') print('NoNameserver, No non-broken nameservers are available to answer the query.')
except dns.resolver.NoAnswer: except dns.resolver.NoAnswer:
self.redis_logger.debug('NoAnswer, The response did not contain an answer to the question.') self.logger.debug('NoAnswer, The response did not contain an answer to the question.')
print('NoAnswer, The response did not contain an answer to the question.') print('NoAnswer, The response did not contain an answer to the question.')
except dns.name.EmptyLabel: except dns.name.EmptyLabel:
self.redis_logger.debug('SyntaxError: EmptyLabel') self.logger.debug('SyntaxError: EmptyLabel')
print('SyntaxError: EmptyLabel') print('SyntaxError: EmptyLabel')
except dns.resolver.NXDOMAIN: except dns.resolver.NXDOMAIN:
# save_mxdomain_in_cache(mxdomain) # save_mxdomain_in_cache(mxdomain)
self.redis_logger.debug('The query name does not exist.') self.logger.debug('The query name does not exist.')
print('The query name does not exist.') print('The query name does not exist.')
except dns.name.LabelTooLong: except dns.name.LabelTooLong:
self.redis_logger.debug('The Label is too long') self.logger.debug('The Label is too long')
print('The Label is too long') print('The Label is too long')
except dns.exception.Timeout: except dns.exception.Timeout:
print('dns timeout') print('dns timeout')
@ -134,9 +134,7 @@ class Mail(AbstractModule):
# # TODO: sanitize mails # # TODO: sanitize mails
def compute(self, message): def compute(self, message):
score = message
item = self.get_obj() item = self.get_obj()
item_date = item.get_date()
mails = self.regex_findall(self.email_regex, item.id, item.get_content()) mails = self.regex_findall(self.email_regex, item.id, item.get_content())
mxdomains_email = {} mxdomains_email = {}
@ -171,15 +169,15 @@ class Mail(AbstractModule):
# for tld in mx_tlds: # for tld in mx_tlds:
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}' msg = f'Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
if num_valid_email > self.mail_threshold: if num_valid_email > self.mail_threshold:
print(f'{item.id} Checked {num_valid_email} e-mail(s)') print(f'{item.id} Checked {num_valid_email} e-mail(s)')
self.redis_logger.warning(msg) self.logger.info(msg)
# Tags # Tags
tag = 'infoleak:automatic-detection="mail"' tag = 'infoleak:automatic-detection="mail"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
elif num_valid_email > 0: elif num_valid_email > 0:
self.redis_logger.info(msg) self.logger.info(msg)
if __name__ == '__main__': if __name__ == '__main__':

17
bin/modules/Onion.py
View file

@ -4,7 +4,7 @@
The Onion Module The Onion Module
============================ ============================
This module extract url from item and returning only ones which are tor This module extract url from object and returning only ones which are tor
related (.onion). All These urls are send to the crawler discovery queue. related (.onion). All These urls are send to the crawler discovery queue.
Requirements Requirements
@ -69,12 +69,11 @@ class Onion(AbstractModule):
onion_urls = [] onion_urls = []
domains = [] domains = []
score = message obj = self.get_obj()
item = self.get_obj() content = obj.get_content()
item_content = item.get_content()
# max execution time on regex # max execution time on regex
res = self.regex_findall(self.onion_regex, item.get_id(), item_content) res = self.regex_findall(self.onion_regex, obj.get_id(), content)
for x in res: for x in res:
# String to tuple # String to tuple
x = x[2:-2].replace(" '", "").split("',") x = x[2:-2].replace(" '", "").split("',")
@ -92,16 +91,14 @@ class Onion(AbstractModule):
if onion_urls: if onion_urls:
if crawlers.is_crawler_activated(): if crawlers.is_crawler_activated():
for domain in domains: for domain in domains:
task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0, task_uuid = crawlers.create_task(domain, parent=obj.get_id(), priority=0,
har=self.har, screenshot=self.screenshot) har=self.har, screenshot=self.screenshot)
if task_uuid: if task_uuid:
print(f'{domain} added to crawler queue: {task_uuid}') print(f'{domain} added to crawler queue: {task_uuid}')
else: else:
to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};' print(f'Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
# TAG Item # TAG Object
tag = 'infoleak:automatic-detection="onion"' tag = 'infoleak:automatic-detection="onion"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

6
bin/modules/Phone.py
View file

@ -62,7 +62,7 @@ class Phone(AbstractModule):
tag = 'infoleak:automatic-detection="phone-number"' tag = 'infoleak:automatic-detection="phone-number"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers') self.logger.info(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
# # List of the regex results in the Item, may be null # # List of the regex results in the Item, may be null
# results = self.REG_PHONE.findall(content) # results = self.REG_PHONE.findall(content)
@ -70,7 +70,7 @@ class Phone(AbstractModule):
# # If the list is greater than 4, we consider the Item may contain a list of phone numbers # # If the list is greater than 4, we consider the Item may contain a list of phone numbers
# if len(results) > 4: # if len(results) > 4:
# self.logger.debug(results) # self.logger.debug(results)
# self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)') # self.logger.info(f'{item.get_id()} contains PID (phone numbers)')
# #
# msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}' # msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
# self.add_message_to_queue(msg, 'Tags') # self.add_message_to_queue(msg, 'Tags')
@ -88,7 +88,7 @@ class Phone(AbstractModule):
# pass # pass
# for country_code in stats: # for country_code in stats:
# if stats[country_code] > 4: # if stats[country_code] > 4:
# self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}') # self.logger.info(f'{item.get_id()} contains Phone numbers with country code {country_code}')
if __name__ == '__main__': if __name__ == '__main__':

17
bin/modules/SQLInjectionDetection.py
View file

@ -16,8 +16,7 @@ import sys
import re import re
import urllib.request import urllib.request
from datetime import datetime # from pyfaup.faup import Faup
from pyfaup.faup import Faup
from urllib.parse import unquote from urllib.parse import unquote
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -25,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader # from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
# from lib import Statistics # from lib import Statistics
class SQLInjectionDetection(AbstractModule): class SQLInjectionDetection(AbstractModule):
@ -39,22 +37,19 @@ class SQLInjectionDetection(AbstractModule):
def __init__(self): def __init__(self):
super(SQLInjectionDetection, self).__init__() super(SQLInjectionDetection, self).__init__()
self.faup = Faup() # self.faup = Faup()
self.logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
url = message url = message
item = self.get_obj()
if self.is_sql_injection(url): if self.is_sql_injection(url):
self.faup.decode(url) # self.faup.decode(url)
url_parsed = self.faup.get() # url_parsed = self.faup.get()
print(f"Detected SQL in URL: {item.id}") print(f"Detected SQL in URL: {self.obj.get_global_id()}")
print(urllib.request.unquote(url)) print(urllib.request.unquote(url))
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print)
# Tag # Tag
tag = f'infoleak:automatic-detection="sql-injection"' tag = f'infoleak:automatic-detection="sql-injection"'

4
bin/modules/SentimentAnalysis.py
View file

@ -167,11 +167,11 @@ class SentimentAnalysis(AbstractModule):
provider_timestamp = provider + '_' + str(timestamp) provider_timestamp = provider + '_' + str(timestamp)
self.db.incr('UniqID') self.db.incr('UniqID')
UniqID = self.db.get('UniqID') UniqID = self.db.get('UniqID')
self.redis_logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines') self.logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
self.db.sadd(provider_timestamp, UniqID) self.db.sadd(provider_timestamp, UniqID)
self.db.set(UniqID, avg_score) self.db.set(UniqID, avg_score)
else: else:
self.redis_logger.debug(f'Dropped:{p_MimeType}') self.logger.debug(f'Dropped:{p_MimeType}')
def isJSON(self, content): def isJSON(self, content):

39
bin/modules/SubmitPaste.py
View file

@ -137,22 +137,21 @@ class SubmitPaste(AbstractModule):
""" """
Create a paste for given file Create a paste for given file
""" """
self.redis_logger.debug('manage')
if os.path.exists(file_full_path): if os.path.exists(file_full_path):
self.redis_logger.debug(f'file exists {file_full_path}') self.logger.debug(f'file exists {file_full_path}')
file_size = os.stat(file_full_path).st_size file_size = os.stat(file_full_path).st_size
self.redis_logger.debug(f'file size {file_size}') self.logger.debug(f'file size {file_size}')
# Verify file length # Verify file length
if file_size < SubmitPaste.FILE_MAX_SIZE: if file_size < SubmitPaste.FILE_MAX_SIZE:
# TODO sanitize filename # TODO sanitize filename
filename = file_full_path.split('/')[-1] filename = file_full_path.split('/')[-1]
self.redis_logger.debug(f'sanitize filename {filename}') self.logger.debug(f'sanitize filename {filename}')
self.redis_logger.debug('file size allowed') self.logger.debug('file size allowed')
if not '.' in filename: if not '.' in filename:
self.redis_logger.debug('no extension for filename') self.logger.debug('no extension for filename')
try: try:
# Read file # Read file
with open(file_full_path,'r') as f: with open(file_full_path,'r') as f:
@ -165,14 +164,14 @@ class SubmitPaste(AbstractModule):
else: else:
file_type = filename.rsplit('.', 1)[1] file_type = filename.rsplit('.', 1)[1]
file_type = file_type.lower() file_type = file_type.lower()
self.redis_logger.debug(f'file ext {file_type}') self.logger.debug(f'file ext {file_type}')
if file_type in SubmitPaste.ALLOWED_EXTENSIONS: if file_type in SubmitPaste.ALLOWED_EXTENSIONS:
self.redis_logger.debug('Extension allowed') self.logger.debug('Extension allowed')
# TODO enum of possible file extension ? # TODO enum of possible file extension ?
# TODO verify file hash with virus total ? # TODO verify file hash with virus total ?
if not self._is_compressed_type(file_type): if not self._is_compressed_type(file_type):
self.redis_logger.debug('Plain text file') self.logger.debug('Plain text file')
# plain txt file # plain txt file
with open(file_full_path,'r') as f: with open(file_full_path,'r') as f:
content = f.read() content = f.read()
@ -197,7 +196,7 @@ class SubmitPaste(AbstractModule):
# except: # except:
# self.abord_file_submission(uuid, "file decompression error") # self.abord_file_submission(uuid, "file decompression error")
# raise # raise
# self.redis_logger.debug('unpacking {} file'.format(files.unpacker)) # self.logger.debug('unpacking {} file'.format(files.unpacker))
# if(not files.children): # if(not files.children):
# self.abord_file_submission(uuid, "Empty compressed file") # self.abord_file_submission(uuid, "Empty compressed file")
# raise # raise
@ -209,11 +208,11 @@ class SubmitPaste(AbstractModule):
# self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source) # self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source)
# n = n + 1 # n = n + 1
# else: # else:
# self.redis_logger.error("Error in module %s: bad extention"%(self.module_name)) # self.logger.error("Error in module %s: bad extention"%(self.module_name))
# self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) ) # self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
# except FileNotFoundError: # except FileNotFoundError:
# self.redis_logger.error("Error in module %s: file not found"%(self.module_name)) # self.logger.error("Error in module %s: file not found"%(self.module_name))
# self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid ) # self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
else: else:
@ -248,7 +247,7 @@ class SubmitPaste(AbstractModule):
# delete uuid # delete uuid
self.r_serv_db.srem('submitted:uuid', uuid) self.r_serv_db.srem('submitted:uuid', uuid)
self.redis_logger.debug(f'{uuid} all file submitted') self.logger.debug(f'{uuid} all file submitted')
print(f'{uuid} all file submitted') print(f'{uuid} all file submitted')
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None): def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
@ -262,11 +261,11 @@ class SubmitPaste(AbstractModule):
full_path = os.path.join(ITEMS_FOLDER, save_path) full_path = os.path.join(ITEMS_FOLDER, save_path)
self.redis_logger.debug(f'file path of the paste {full_path}') self.logger.debug(f'file path of the paste {full_path}')
if not os.path.isfile(full_path): if not os.path.isfile(full_path):
# file not exists in AIL paste directory # file not exists in AIL paste directory
self.redis_logger.debug(f"new paste {paste_content}") self.logger.debug(f"new paste {paste_content}")
gzip64encoded = self._compress_encode_content(paste_content, uuid) gzip64encoded = self._compress_encode_content(paste_content, uuid)
@ -274,7 +273,7 @@ class SubmitPaste(AbstractModule):
# use relative path # use relative path
rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1) rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1)
self.redis_logger.debug(f"relative path {rel_item_path}") self.logger.debug(f"relative path {rel_item_path}")
item = Item(rel_item_path) item = Item(rel_item_path)
@ -295,12 +294,12 @@ class SubmitPaste(AbstractModule):
if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'): if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'):
self.r_serv_log_submit.set(f'{uuid}:end', 1) self.r_serv_log_submit.set(f'{uuid}:end', 1)
self.redis_logger.debug(f' {rel_item_path} send to Mixer') self.logger.debug(f' {rel_item_path} send to Mixer')
print(f' {rel_item_path} send to Mixer') print(f' {rel_item_path} send to Mixer')
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today() curr_date = datetime.date.today()
self.redis_logger.debug("paste submitted") self.logger.debug("paste submitted")
else: else:
self.addError(uuid, f'File: {save_path} already exist in submitted pastes') self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
@ -316,7 +315,7 @@ class SubmitPaste(AbstractModule):
return gzip64encoded return gzip64encoded
def addError(self, uuid, errorMessage): def addError(self, uuid, errorMessage):
self.redis_logger.debug(errorMessage) self.logger.debug(errorMessage)
print(errorMessage) print(errorMessage)
error = self.r_serv_log_submit.get(f'{uuid}:error') error = self.r_serv_log_submit.get(f'{uuid}:error')
if error is not None: if error is not None:
@ -324,7 +323,7 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.incr(f'{uuid}:nb_end') self.r_serv_log_submit.incr(f'{uuid}:nb_end')
def abord_file_submission(self, uuid, errorMessage): def abord_file_submission(self, uuid, errorMessage):
self.redis_logger.debug(f'abord {uuid}, {errorMessage}') self.logger.debug(f'abord {uuid}, {errorMessage}')
self.addError(uuid, errorMessage) self.addError(uuid, errorMessage)
self.r_serv_log_submit.set(f'{uuid}:end', 1) self.r_serv_log_submit.set(f'{uuid}:end', 1)

2
bin/modules/Urls.py
View file

@ -85,8 +85,6 @@ class Urls(AbstractModule):
if len(l_urls) > 0: if len(l_urls) > 0:
to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(to_print) print(to_print)
# .debug ???
# self.redis_logger.info(f'{to_print}Detected {len(l_urls)} URL;{item.get_id()}')
if __name__ == '__main__': if __name__ == '__main__':

10
bin/modules/abstract_module.py
View file

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from pubsublogger import publisher
from lib import ail_logger from lib import ail_logger
from lib.ail_queues import AILQueue from lib.ail_queues import AILQueue
from lib import regex_helper from lib import regex_helper
@ -51,15 +50,6 @@ class AbstractModule(ABC):
self.obj = None self.obj = None
self.sha256_mess = None self.sha256_mess = None
# Init Redis Logger
self.redis_logger = publisher
# Port of the redis instance used by pubsublogger
self.redis_logger.port = 6380
# Channel name to publish logs
# # TODO: refactor logging
# If provided could be a namespaced channel like script:<ModuleName>
self.redis_logger.channel = 'Script'
# Cache key # Cache key
self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name) self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
self.max_execution_time = 30 self.max_execution_time = 30

12
bin/trackers/Retro_Hunt.py
View file

@ -42,12 +42,11 @@ class Retro_Hunt_Module(AbstractModule):
self.obj = None self.obj = None
self.tags = [] self.tags = []
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
# # TODO: # start_time # # TODO: # start_time
# # end_time # # end_time
def compute(self, task_uuid): def compute(self, task_uuid):
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
print(f'starting Retro hunt task {task_uuid}') print(f'starting Retro hunt task {task_uuid}')
self.progress = 0 self.progress = 0
# First launch # First launch
@ -58,7 +57,7 @@ class Retro_Hunt_Module(AbstractModule):
timeout = self.retro_hunt.get_timeout() timeout = self.retro_hunt.get_timeout()
self.tags = self.retro_hunt.get_tags() self.tags = self.retro_hunt.get_tags()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') self.logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
# Filters # Filters
filters = self.retro_hunt.get_filters() filters = self.retro_hunt.get_filters()
@ -111,7 +110,6 @@ class Retro_Hunt_Module(AbstractModule):
# Completed # Completed
self.retro_hunt.complete() self.retro_hunt.complete()
print(f'Retro Hunt {task_uuid} completed') print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
def update_progress(self): def update_progress(self):
if self.nb_objs == 0: if self.nb_objs == 0:
@ -128,7 +126,6 @@ class Retro_Hunt_Module(AbstractModule):
# print(data) # print(data)
task_uuid = data['namespace'] task_uuid = data['namespace']
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}') print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id) self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id)
@ -158,16 +155,15 @@ class Retro_Hunt_Module(AbstractModule):
task_uuid = Tracker.get_retro_hunt_task_to_start() task_uuid = Tracker.get_retro_hunt_task_to_start()
if task_uuid: if task_uuid:
# Module processing with the message from the queue # Module processing with the message from the queue
self.redis_logger.debug(task_uuid) self.logger.debug(task_uuid)
# try: # try:
self.compute(task_uuid) self.compute(task_uuid)
# except Exception as err: # except Exception as err:
# self.redis_logger.error(f'Error in module {self.module_name}: {err}') # self.logger.error(f'Error in module {self.module_name}: {err}')
# # Remove uuid ref # # Remove uuid ref
# self.remove_submit_uuid(uuid) # self.remove_submit_uuid(uuid)
else: else:
# Wait before next process # Wait before next process
self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s')
time.sleep(self.pending_seconds) time.sleep(self.pending_seconds)

4
bin/trackers/Tracker_Regex.py
View file

@ -47,14 +47,13 @@ class Tracker_Regex(AbstractModule):
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()} 'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
# refresh Tracked regex # refresh Tracked regex
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.tracked_regexs = Tracker.get_tracked_regexs() self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time() self.last_refresh = time.time()
self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed') print('Tracked regex refreshed')
obj = self.get_obj() obj = self.get_obj()
@ -117,7 +116,6 @@ class Tracker_Regex(AbstractModule):
continue continue
print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

7
bin/trackers/Tracker_Term.py
View file

@ -62,20 +62,18 @@ class Tracker_Term(AbstractModule):
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()} 'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
# refresh Tracked term # refresh Tracked term
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'): if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
self.tracked_words = Tracker.get_tracked_words() self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time() self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed') print('Tracked word refreshed')
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'): if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
self.tracked_sets = Tracker.get_tracked_sets() self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time() self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed') print('Tracked set refreshed')
obj = self.get_obj() obj = self.get_obj()
@ -93,7 +91,7 @@ class Tracker_Term(AbstractModule):
try: try:
dict_words_freq = Tracker.get_text_word_frequency(content) dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException: except TimeoutException:
self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout") self.logger.warning(f"{self.obj.get_global_id()} processing timeout")
else: else:
signal.alarm(0) signal.alarm(0)
@ -125,7 +123,6 @@ class Tracker_Term(AbstractModule):
continue continue
print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}') print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(), obj_id) tracker.add(obj.get_type(), obj.get_subtype(), obj_id)

4
bin/trackers/Tracker_Typo_Squatting.py
View file

@ -43,14 +43,13 @@ class Tracker_Typo_Squatting(AbstractModule):
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()} 'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
# refresh Tracked typo # refresh Tracked typo
if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'): if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
self.tracked_typosquattings = Tracker.get_tracked_typosquatting() self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquatting = time.time() self.last_refresh_typosquatting = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed')
host = message host = message
@ -76,7 +75,6 @@ class Tracker_Typo_Squatting(AbstractModule):
continue continue
print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}') print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

5
bin/trackers/Tracker_Yara.py
View file

@ -44,14 +44,13 @@ class Tracker_Yara(AbstractModule):
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()} 'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
# refresh YARA list # refresh YARA list
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
self.rules = Tracker.get_tracked_yara_rules() self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time() self.last_refresh = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed') print('Tracked set refreshed')
self.obj = self.get_obj() self.obj = self.get_obj()
@ -69,11 +68,9 @@ class Tracker_Yara(AbstractModule):
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60) which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match: if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}')
print(f'{self.obj.get_global_id()}: {yara_match}') print(f'{self.obj.get_global_id()}: {yara_match}')
except yara.TimeoutError: except yara.TimeoutError:
print(f'{self.obj.get_id()}: yara scanning timed out') print(f'{self.obj.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
def convert_byte_offset_to_string(self, b_content, offset): def convert_byte_offset_to_string(self, b_content, offset):
byte_chunk = b_content[:offset + 1] byte_chunk = b_content[:offset + 1]

3
requirements.txt
View file

@ -25,9 +25,6 @@ xxhash>3.1.0
zmq zmq
pyzmq>19.0.0 pyzmq>19.0.0
# Logging
logbook
pubsublogger
# Tokeniser # Tokeniser
nltk>3.4.5 nltk>3.4.5

8
var/www/Flask_config.py
View file

@ -16,7 +16,6 @@ import sys
################################## ##################################
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader from lib import ConfigLoader
from pubsublogger import publisher
# FLASK # # FLASK #
app = None app = None
@ -34,13 +33,6 @@ r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint
r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint
# Logger (Redis)
redis_logger = publisher
# Port of the redis instance used by pubsublogger
redis_logger.port = 6380
# Channel name to publish logs
redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')
#### VARIABLES #### #### VARIABLES ####

27
var/www/blueprints/ui_submit.py
View file

@ -39,8 +39,6 @@ baseUrl = Flask_config.baseUrl
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
logger = Flask_config.redis_logger
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits) valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
@ -57,11 +55,9 @@ def limit_content_length():
def decorator(f): def decorator(f):
@wraps(f) @wraps(f)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
logger.debug('decorator')
cl = request.content_length cl = request.content_length
if cl is not None: if cl is not None:
if cl > Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE or ('file' not in request.files and cl > Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE): if cl > Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE or ('file' not in request.files and cl > Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE):
logger.debug('abort')
abort(413) abort(413)
return f(*args, **kwargs) return f(*args, **kwargs)
return wrapper return wrapper
@ -75,7 +71,6 @@ def allowed_file(filename):
return True return True
else: else:
file_ext = filename.rsplit('.', 1)[1].lower() file_ext = filename.rsplit('.', 1)[1].lower()
logger.debug(file_ext)
return file_ext in Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS return file_ext in Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
@ -111,7 +106,6 @@ def PasteSubmit_page():
@login_user_no_api @login_user_no_api
@limit_content_length() @limit_content_length()
def submit(): def submit():
logger.debug('submit')
password = request.form['archive_pass'] password = request.form['archive_pass']
ltags = request.form['tags_taxonomies'] ltags = request.form['tags_taxonomies']
@ -124,12 +118,10 @@ def submit():
paste_source = paste_source.replace('/', '')[:80] paste_source = paste_source.replace('/', '')[:80]
if paste_source in ['crawled', 'tests']: if paste_source in ['crawled', 'tests']:
content = 'Invalid source' content = 'Invalid source'
logger.info(paste_source)
return content, 400 return content, 400
if not re.match('^[0-9a-zA-Z-_\+@#&\.;=:!]*$', paste_source): if not re.match('^[0-9a-zA-Z-_\+@#&\.;=:!]*$', paste_source):
content = f'Invalid source name: Forbidden character(s)' content = f'Invalid source name: Forbidden character(s)'
logger.info(content)
return content, 400 return content, 400
is_file = False is_file = False
@ -139,8 +131,6 @@ def submit():
if file_import.filename: if file_import.filename:
is_file = True is_file = True
logger.debug(f'is file ? {is_file}')
submitted_tag = 'infoleak:submission="manual"' submitted_tag = 'infoleak:submission="manual"'
# active taxonomies # active taxonomies
@ -149,13 +139,11 @@ def submit():
active_galaxies = Tag.get_active_galaxies() active_galaxies = Tag.get_active_galaxies()
if ltags or ltagsgalaxies: if ltags or ltagsgalaxies:
logger.debug(f'ltags ? {ltags} {ltagsgalaxies}')
ltags = Tag.unpack_str_tags_list(ltags) ltags = Tag.unpack_str_tags_list(ltags)
ltagsgalaxies = Tag.unpack_str_tags_list(ltagsgalaxies) ltagsgalaxies = Tag.unpack_str_tags_list(ltagsgalaxies)
if not Tag.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies): if not Tag.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
content = 'INVALID TAGS' content = 'INVALID TAGS'
logger.info(content)
return content, 400 return content, 400
# add submitted tags # add submitted tags
@ -164,36 +152,28 @@ def submit():
ltags.append(submitted_tag) ltags.append(submitted_tag)
if is_file: if is_file:
logger.debug('file management')
if allowed_file(file_import.filename): if allowed_file(file_import.filename):
logger.debug('file extension allowed')
# get UUID # get UUID
UUID = str(uuid.uuid4()) UUID = str(uuid.uuid4())
# create submitted dir # create submitted dir
if not os.path.exists(UPLOAD_FOLDER): if not os.path.exists(UPLOAD_FOLDER):
logger.debug('create folder')
os.makedirs(UPLOAD_FOLDER) os.makedirs(UPLOAD_FOLDER)
if '.' not in file_import.filename: if '.' not in file_import.filename:
logger.debug('add UUID to path')
full_path = os.path.join(UPLOAD_FOLDER, UUID) full_path = os.path.join(UPLOAD_FOLDER, UUID)
else: else:
if file_import.filename[-6:] == 'tar.gz': if file_import.filename[-6:] == 'tar.gz':
logger.debug('file extension is tar.gz')
file_type = 'tar.gz' file_type = 'tar.gz'
else: else:
file_type = file_import.filename.rsplit('.', 1)[1] file_type = file_import.filename.rsplit('.', 1)[1]
logger.debug(f'file type {file_type}')
name = UUID + '.' + file_type name = UUID + '.' + file_type
full_path = os.path.join(UPLOAD_FOLDER, name) full_path = os.path.join(UPLOAD_FOLDER, name)
logger.debug(f'full path {full_path}')
# Flask verify the file size # Flask verify the file size
file_import.save(full_path) file_import.save(full_path)
logger.debug('file saved')
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True) Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
@ -204,18 +184,13 @@ def submit():
else: else:
content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension' content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension'
logger.info(content)
return content, 400 return content, 400
elif paste_content != '': elif paste_content != '':
logger.debug(f'entering text paste management')
if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE: if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE:
logger.debug(f'size {sys.getsizeof(paste_content)}')
# get id # get id
UUID = str(uuid.uuid4()) UUID = str(uuid.uuid4())
logger.debug('create import')
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, source=paste_source) Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, source=paste_source)
logger.debug('import OK')
return render_template("submit_items.html", return render_template("submit_items.html",
active_taxonomies = active_taxonomies, active_taxonomies = active_taxonomies,
active_galaxies = active_galaxies, active_galaxies = active_galaxies,
@ -223,11 +198,9 @@ def submit():
else: else:
content = f'text paste size is over {Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE} bytes limit' content = f'text paste size is over {Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE} bytes limit'
logger.info(content)
return content, 400 return content, 400
content = 'submit aborded' content = 'submit aborded'
logger.error(content)
return content, 400 return content, 400
return PasteSubmit_page() return PasteSubmit_page()