AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 00:16:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chg: [pubsublogger] remove old redis_logger

Browse source
This commit is contained in:
terrtia 2024-12-09 13:54:54 +01:00
parent feabe9f5df
commit 849977f0c2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
32 changed files with 112 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
bin/LAUNCH.sh
View file

@ -615,14 +615,13 @@ function launch_all {
update;
launch_redis;
launch_kvrocks;
launch_logs;
launch_scripts;
launch_flask;
}
function menu_display {
options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
menu() {
echo "What do you want to Launch?:"
@ -653,9 +652,6 @@ function menu_display {
Kvrocks)
launch_kvrocks;
;;
Logs)
launch_logs;
;;
Scripts)
launch_scripts;
;;

1
bin/core/Sync_module.py
View file

@ -88,7 +88,6 @@ class Sync_module(AbstractModule):
if self.last_refresh_queues < time.time():
timeout_processed_objs()
self.last_refresh_queues = time.time() + 120
self.redis_logger.debug('Timeout queues')
# print('Timeout queues')
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)

16
bin/modules/ApiKey.py
View file

@ -20,7 +20,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class ApiKey(AbstractModule):
"""ApiKey module for AIL framework"""
@ -48,20 +47,19 @@ class ApiKey(AbstractModule):
def compute(self, message, r_result=False):
score = message
item = self.get_obj()
item_content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True)
if aws_access_key:
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True)
if aws_access_key or google_api_key:
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
to_print = obj.get_global_id()
if google_api_key:
print(f'found google api key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="google-api-key"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -69,10 +67,8 @@ class ApiKey(AbstractModule):
# # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
if aws_access_key:
print(f'found AWS key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
if aws_secret_key:
print(f'found AWS secret key')
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="aws-key"'
self.add_message_to_queue(message=tag, queue='Tags')

5
bin/modules/Categ.py
View file

@ -65,7 +65,7 @@ class Categ(AbstractModule):
self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
self.reload_categ_words()
self.redis_logger.info("Script Categ started")
self.logger.info("Script Categ started")
# # TODO: trigger reload on change ( save last reload time, ...)
def reload_categ_words(self):
@ -103,9 +103,6 @@ class Categ(AbstractModule):
print(msg, categ)
self.add_message_to_queue(message=msg, queue=categ)
self.redis_logger.debug(
f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
if r_result:
return categ_found

22
bin/modules/Credential.py
View file

@ -29,7 +29,6 @@ Redis organization:
import os
import sys
import time
from datetime import datetime
from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN'])
@ -37,9 +36,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib import ConfigLoader
# from lib import Statistics
class Credential(AbstractModule):
@ -80,40 +77,36 @@ class Credential(AbstractModule):
self.pending_seconds = 10
# Send module state to logs
self.redis_logger.info(f"Module {self.module_name} initialized")
self.logger.info(f"Module {self.module_name} initialized")
def compute(self, message):
count = message
item = self.get_obj()
obj = self.get_obj()
item_content = item.get_content()
content = obj.get_content()
# TODO: USE SETS
# Extract all credentials
all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content)
all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content)
if all_credentials:
nb_cred = len(all_credentials)
message = f'Checked {nb_cred} credentials found.'
all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True)
all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True)
if all_sites:
discovered_sites = ', '.join(all_sites)
message += f' Related websites: {discovered_sites}'
print(message)
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
# num of creds above threshold, publish an alert
if nb_cred > self.criticalNumberToAlert:
print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
self.redis_logger.warning(to_print)
tag = 'infoleak:automatic-detection="credential"'
self.add_message_to_queue(message=tag, queue='Tags')
site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content)
site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content)
creds_sites = {}
@ -162,8 +155,7 @@ class Credential(AbstractModule):
# for tld in nb_tlds:
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
else:
self.redis_logger.info(to_print)
print(f'found {nb_cred} credentials')
print(f'found {nb_cred} credentials {self.obj.get_global_id()}')
# # TODO: # FIXME: TEMP DESABLE
# # For searching credential in termFreq

17
bin/modules/CreditCards.py
View file

@ -7,7 +7,7 @@ The CreditCards Module
This module is consuming the Redis-list created by the Categ module.
It apply credit card regexes on item content and warn if a valid card number is found.
It apply credit card regexes on object content and warn if a valid card number is found.
"""
@ -23,7 +23,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from packages import lib_refine
class CreditCards(AbstractModule):
@ -68,10 +67,9 @@ class CreditCards(AbstractModule):
return extracted
def compute(self, message, r_result=False):
score = message
item = self.get_obj()
content = item.get_content()
all_cards = self.regex_findall(self.regex, item.id, content)
obj = self.get_obj()
content = obj.get_content()
all_cards = self.regex_findall(self.regex, obj.id, content)
if len(all_cards) > 0:
# self.logger.debug(f'All matching {all_cards}')
@ -84,11 +82,8 @@ class CreditCards(AbstractModule):
creditcard_set.add(valid_card)
# print(creditcard_set)
to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
if creditcard_set:
mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}'
print(mess)
self.redis_logger.warning(mess)
print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="credit-card"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -96,7 +91,7 @@ class CreditCards(AbstractModule):
if r_result:
return creditcard_set
else:
self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}')
print(f'CreditCard related;{self.obj.get_global_id()}')
if __name__ == '__main__':

10
bin/modules/Cryptocurrencies.py
View file

@ -28,7 +28,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.objects.CryptoCurrencies import CryptoCurrency
from lib.objects.Items import Item
##################################
##################################
@ -151,17 +150,10 @@ class Cryptocurrencies(AbstractModule, ABC):
# debug
print(private_keys)
to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(),
item.get_date(),
item.get_basename())
self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
else:
private_keys = []
to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys"
print(to_print)
self.redis_logger.warning(to_print)
print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}")
if __name__ == '__main__':

5
bin/modules/CveModule.py
View file

@ -44,7 +44,6 @@ class CveModule(AbstractModule):
self.logger.info(f'Module {self.module_name} initialized')
def compute(self, message):
count = message
item = self.get_obj()
item_id = item.get_id()
@ -56,9 +55,7 @@ class CveModule(AbstractModule):
cve = Cves.Cve(cve_id)
cve.add(date, item)
warning = f'{self.obj.get_global_id()} contains CVEs {cves}'
print(warning)
self.redis_logger.warning(warning)
print(f'{self.obj.get_global_id()} contains CVEs {cves}')
tag = 'infoleak:automatic-detection="cve"'
# Send to Tags Queue

10
bin/modules/DomClassifier.py
View file

@ -58,10 +58,6 @@ class DomClassifier(AbstractModule):
def compute(self, message, r_result=False):
host = message
item = self.get_obj()
item_basename = item.get_basename()
item_date = item.get_date()
item_source = item.get_source()
try:
self.dom_classifier.text(rawtext=host)
@ -82,19 +78,19 @@ class DomClassifier(AbstractModule):
localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
if self.cc:
localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
if r_result:
return self.dom_classifier.vdomain
except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed")
raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")

5
bin/modules/Duplicates.py
View file

@ -27,7 +27,6 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib import Duplicate
from lib.objects.Items import Item
class Duplicates(AbstractModule):
@ -92,11 +91,11 @@ class Duplicates(AbstractModule):
Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
if nb_duplicates:
self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}')
self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}')
y = time.time()
print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
# self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x))
# self.logger.debug('{}Processed in {} sec'.format(to_print, y-x))
if __name__ == "__main__":

1
bin/modules/Global.py
View file

@ -74,7 +74,6 @@ class Global(AbstractModule):
if int(difftime) > 30:
to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s'
print(to_print)
self.redis_logger.debug(to_print)
self.time_last_stats = time.time()
self.processed_item = 0

9
bin/modules/IPAddress.py
View file

@ -25,7 +25,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import regex_helper
# TODO REWRITE ME -> PERF + IPV6 + Tracker ?
@ -66,11 +65,11 @@ class IPAddress(AbstractModule):
if not self.ip_networks:
return None
item = self.get_obj()
content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# list of the regex results in the Item
results = self.regex_findall(self.re_ipv4, item.get_id(), content)
results = self.regex_findall(self.re_ipv4, obj.get_id(), content)
results = set(results)
matching_ips = []
for ip in results:
@ -83,7 +82,6 @@ class IPAddress(AbstractModule):
if len(matching_ips) > 0:
self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
# Tag message with IP
tag = 'infoleak:automatic-detection="ip"'
@ -93,4 +91,3 @@ class IPAddress(AbstractModule):
if __name__ == "__main__":
module = IPAddress()
module.run()
# module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')

21
bin/modules/Iban.py
View file

@ -24,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.ConfigLoader import ConfigLoader
# from lib.ConfigLoader import ConfigLoader
# from lib import Statistics
class Iban(AbstractModule):
@ -40,7 +39,7 @@ class Iban(AbstractModule):
def __init__(self, queue=True):
super(Iban, self).__init__(queue=queue)
# Waiting time in secondes between to message proccessed
# Waiting time in seconds between to message processed
self.pending_seconds = 10
self.regex_timeout = 30
@ -49,7 +48,7 @@ class Iban(AbstractModule):
self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
# Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized')
self.logger.info(f'Module {self.module_name} initialized')
def get_iban_number(self, iban):
return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN)
@ -73,29 +72,27 @@ class Iban(AbstractModule):
return extracted
def compute(self, message):
item = self.get_obj()
item_id = item.get_id()
obj = self.get_obj()
obj_id = obj.get_id()
ibans = self.regex_findall(self.iban_regex, item_id, item.get_content())
ibans = self.regex_findall(self.iban_regex, obj_id, obj.get_content())
if ibans:
valid_ibans = set()
for iban in ibans:
iban = iban[1:-1].replace("'", "").split(',')
iban = iban[0]+iban[1]+iban[2]
iban = ''.join(e for e in iban if e.isalnum())
if self.regex_findall(self.iban_regex_verify, item_id, iban):
if self.regex_findall(self.iban_regex_verify, obj_id, iban):
print(f'checking {iban}')
if self.is_valid_iban(iban):
valid_ibans.add(iban)
if valid_ibans:
print(f'{valid_ibans} ibans {item_id}')
date = datetime.datetime.now().strftime("%Y%m")
print(f'{valid_ibans} ibans {self.obj.get_global_id()}')
# date = datetime.datetime.now().strftime("%Y%m")
# for iban in valid_ibans:
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}')
# Tags
tag = 'infoleak:automatic-detection="iban"'
self.add_message_to_queue(message=tag, queue='Tags')

29
bin/modules/Keys.py
View file

@ -24,7 +24,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class KeyEnum(Enum):
@ -56,14 +55,14 @@ class Keys(AbstractModule):
self.pending_seconds = 1
def compute(self, message):
item = self.get_obj()
content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# find = False
get_pgp_content = False
if KeyEnum.PGP_MESSAGE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message')
print(f'{self.obj.get_global_id()} has a PGP enc message')
tag = 'infoleak:automatic-detection="pgp-message"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -81,21 +80,21 @@ class Keys(AbstractModule):
get_pgp_content = True
if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message')
print(f'{self.obj.get_global_id()} has a pgp private key block message')
tag = 'infoleak:automatic-detection="pgp-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
get_pgp_content = True
if KeyEnum.CERTIFICATE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message')
print(f'{self.obj.get_global_id()} has a certificate message')
tag = 'infoleak:automatic-detection="certificate"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.RSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message')
print(f'{self.obj.get_global_id()} has a RSA private key message')
print('rsa private key message found')
tag = 'infoleak:automatic-detection="rsa-private-key"'
@ -103,7 +102,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message')
print(f'{self.obj.get_global_id()} has a private key message')
print('private key message found')
tag = 'infoleak:automatic-detection="private-key"'
@ -111,7 +110,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message')
print(f'{self.obj.get_global_id()} has an encrypted private key message')
print('encrypted private key message found')
tag = 'infoleak:automatic-detection="encrypted-private-key"'
@ -119,7 +118,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print(f'{self.obj.get_global_id()} has an openssh private key message')
print('openssh private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -127,7 +126,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message')
print(f'{self.obj.get_global_id()} has an ssh2 private key message')
print('SSH2 private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -135,7 +134,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print(f'{self.obj.get_global_id()} has an openssh private key message')
print('OpenVPN Static key message found')
tag = 'infoleak:automatic-detection="vpn-static-key"'
@ -143,21 +142,21 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.DSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message')
print(f'{self.obj.get_global_id()} has a dsa private key message')
tag = 'infoleak:automatic-detection="dsa-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.EC_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message')
print(f'{self.obj.get_global_id()} has an ec private key message')
tag = 'infoleak:automatic-detection="ec-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.PUBLIC_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message')
print(f'{self.obj.get_global_id()} has a public key message')
tag = 'infoleak:automatic-detection="public-key"'
self.add_message_to_queue(message=tag, queue='Tags')

10
bin/modules/LibInjection.py
View file

@ -34,10 +34,9 @@ class LibInjection(AbstractModule):
self.faup = Faup()
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
item = self.get_obj()
url = message
self.faup.decode(url)
@ -66,12 +65,9 @@ class LibInjection(AbstractModule):
# print(f'query is sqli : {result_query}')
if result_path['sqli'] is True or result_query['sqli'] is True:
item_id = item.get_id()
print(f"Detected (libinjection) SQL in URL: {item_id}")
print(unquote(url))
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print)
self.logger.info(f'Detected SQL in URL;{self.obj.get_global_id()}')
print(unquote(url))
# Add tag
tag = 'infoleak:automatic-detection="sql-injection"'

3
bin/modules/MISP_Thehive_Auto_Push.py
View file

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.exceptions import MISPConnectionError
from lib.objects.Items import Item
from lib import Tag
from exporter.MISPExporter import MISPExporterAutoDaily
from exporter.TheHiveExporter import TheHiveExporterAlertTag
@ -43,7 +42,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0:
self.tags = Tag.refresh_auto_push()
self.last_refresh = time.time()
self.redis_logger.info('Tags Auto Push refreshed')
self.logger.debug('Tags Auto Push refreshed')
tag = message
item = self.get_obj()

18
bin/modules/Mail.py
View file

@ -95,20 +95,20 @@ class Mail(AbstractModule):
# print()
except dns.resolver.NoNameservers:
self.redis_logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
self.logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
print('NoNameserver, No non-broken nameservers are available to answer the query.')
except dns.resolver.NoAnswer:
self.redis_logger.debug('NoAnswer, The response did not contain an answer to the question.')
self.logger.debug('NoAnswer, The response did not contain an answer to the question.')
print('NoAnswer, The response did not contain an answer to the question.')
except dns.name.EmptyLabel:
self.redis_logger.debug('SyntaxError: EmptyLabel')
self.logger.debug('SyntaxError: EmptyLabel')
print('SyntaxError: EmptyLabel')
except dns.resolver.NXDOMAIN:
# save_mxdomain_in_cache(mxdomain)
self.redis_logger.debug('The query name does not exist.')
self.logger.debug('The query name does not exist.')
print('The query name does not exist.')
except dns.name.LabelTooLong:
self.redis_logger.debug('The Label is too long')
self.logger.debug('The Label is too long')
print('The Label is too long')
except dns.exception.Timeout:
print('dns timeout')
@ -134,9 +134,7 @@ class Mail(AbstractModule):
# # TODO: sanitize mails
def compute(self, message):
score = message
item = self.get_obj()
item_date = item.get_date()
mails = self.regex_findall(self.email_regex, item.id, item.get_content())
mxdomains_email = {}
@ -171,15 +169,15 @@ class Mail(AbstractModule):
# for tld in mx_tlds:
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
msg = f'Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
if num_valid_email > self.mail_threshold:
print(f'{item.id} Checked {num_valid_email} e-mail(s)')
self.redis_logger.warning(msg)
self.logger.info(msg)
# Tags
tag = 'infoleak:automatic-detection="mail"'
self.add_message_to_queue(message=tag, queue='Tags')
elif num_valid_email > 0:
self.redis_logger.info(msg)
self.logger.info(msg)
if __name__ == '__main__':

17
bin/modules/Onion.py
View file

@ -4,7 +4,7 @@
The Onion Module
============================
This module extract url from item and returning only ones which are tor
This module extract url from object and returning only ones which are tor
related (.onion). All These urls are send to the crawler discovery queue.
Requirements
@ -69,12 +69,11 @@ class Onion(AbstractModule):
onion_urls = []
domains = []
score = message
item = self.get_obj()
item_content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# max execution time on regex
res = self.regex_findall(self.onion_regex, item.get_id(), item_content)
res = self.regex_findall(self.onion_regex, obj.get_id(), content)
for x in res:
# String to tuple
x = x[2:-2].replace(" '", "").split("',")
@ -92,16 +91,14 @@ class Onion(AbstractModule):
if onion_urls:
if crawlers.is_crawler_activated():
for domain in domains:
task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0,
task_uuid = crawlers.create_task(domain, parent=obj.get_id(), priority=0,
har=self.har, screenshot=self.screenshot)
if task_uuid:
print(f'{domain} added to crawler queue: {task_uuid}')
else:
to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
print(f'Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
# TAG Item
# TAG Object
tag = 'infoleak:automatic-detection="onion"'
self.add_message_to_queue(message=tag, queue='Tags')

6
bin/modules/Phone.py
View file

@ -62,7 +62,7 @@ class Phone(AbstractModule):
tag = 'infoleak:automatic-detection="phone-number"'
self.add_message_to_queue(message=tag, queue='Tags')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
self.logger.info(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
# # List of the regex results in the Item, may be null
# results = self.REG_PHONE.findall(content)
@ -70,7 +70,7 @@ class Phone(AbstractModule):
# # If the list is greater than 4, we consider the Item may contain a list of phone numbers
# if len(results) > 4:
# self.logger.debug(results)
# self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
# self.logger.info(f'{item.get_id()} contains PID (phone numbers)')
#
# msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
# self.add_message_to_queue(msg, 'Tags')
@ -88,7 +88,7 @@ class Phone(AbstractModule):
# pass
# for country_code in stats:
# if stats[country_code] > 4:
# self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
# self.logger.info(f'{item.get_id()} contains Phone numbers with country code {country_code}')
if __name__ == '__main__':

17
bin/modules/SQLInjectionDetection.py
View file

@ -16,8 +16,7 @@ import sys
import re
import urllib.request
from datetime import datetime
from pyfaup.faup import Faup
# from pyfaup.faup import Faup
from urllib.parse import unquote
sys.path.append(os.environ['AIL_BIN'])
@ -25,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
# from lib.ConfigLoader import ConfigLoader
# from lib import Statistics
class SQLInjectionDetection(AbstractModule):
@ -39,22 +37,19 @@ class SQLInjectionDetection(AbstractModule):
def __init__(self):
super(SQLInjectionDetection, self).__init__()
self.faup = Faup()
# self.faup = Faup()
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
url = message
item = self.get_obj()
if self.is_sql_injection(url):
self.faup.decode(url)
url_parsed = self.faup.get()
# self.faup.decode(url)
# url_parsed = self.faup.get()
print(f"Detected SQL in URL: {item.id}")
print(f"Detected SQL in URL: {self.obj.get_global_id()}")
print(urllib.request.unquote(url))
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print)
# Tag
tag = f'infoleak:automatic-detection="sql-injection"'

4
bin/modules/SentimentAnalysis.py
View file

@ -167,11 +167,11 @@ class SentimentAnalysis(AbstractModule):
provider_timestamp = provider + '_' + str(timestamp)
self.db.incr('UniqID')
UniqID = self.db.get('UniqID')
self.redis_logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
self.logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
self.db.sadd(provider_timestamp, UniqID)
self.db.set(UniqID, avg_score)
else:
self.redis_logger.debug(f'Dropped:{p_MimeType}')
self.logger.debug(f'Dropped:{p_MimeType}')
def isJSON(self, content):

39
bin/modules/SubmitPaste.py
View file

@ -137,22 +137,21 @@ class SubmitPaste(AbstractModule):
"""
Create a paste for given file
"""
self.redis_logger.debug('manage')
if os.path.exists(file_full_path):
self.redis_logger.debug(f'file exists {file_full_path}')
self.logger.debug(f'file exists {file_full_path}')
file_size = os.stat(file_full_path).st_size
self.redis_logger.debug(f'file size {file_size}')
self.logger.debug(f'file size {file_size}')
# Verify file length
if file_size < SubmitPaste.FILE_MAX_SIZE:
# TODO sanitize filename
filename = file_full_path.split('/')[-1]
self.redis_logger.debug(f'sanitize filename {filename}')
self.redis_logger.debug('file size allowed')
self.logger.debug(f'sanitize filename {filename}')
self.logger.debug('file size allowed')
if not '.' in filename:
self.redis_logger.debug('no extension for filename')
self.logger.debug('no extension for filename')
try:
# Read file
with open(file_full_path,'r') as f:
@ -165,14 +164,14 @@ class SubmitPaste(AbstractModule):
else:
file_type = filename.rsplit('.', 1)[1]
file_type = file_type.lower()
self.redis_logger.debug(f'file ext {file_type}')
self.logger.debug(f'file ext {file_type}')
if file_type in SubmitPaste.ALLOWED_EXTENSIONS:
self.redis_logger.debug('Extension allowed')
self.logger.debug('Extension allowed')
# TODO enum of possible file extension ?
# TODO verify file hash with virus total ?
if not self._is_compressed_type(file_type):
self.redis_logger.debug('Plain text file')
self.logger.debug('Plain text file')
# plain txt file
with open(file_full_path,'r') as f:
content = f.read()
@ -197,7 +196,7 @@ class SubmitPaste(AbstractModule):
# except:
# self.abord_file_submission(uuid, "file decompression error")
# raise
# self.redis_logger.debug('unpacking {} file'.format(files.unpacker))
# self.logger.debug('unpacking {} file'.format(files.unpacker))
# if(not files.children):
# self.abord_file_submission(uuid, "Empty compressed file")
# raise
@ -209,11 +208,11 @@ class SubmitPaste(AbstractModule):
# self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source)
# n = n + 1
# else:
# self.redis_logger.error("Error in module %s: bad extention"%(self.module_name))
# self.logger.error("Error in module %s: bad extention"%(self.module_name))
# self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
# except FileNotFoundError:
# self.redis_logger.error("Error in module %s: file not found"%(self.module_name))
# self.logger.error("Error in module %s: file not found"%(self.module_name))
# self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
else:
@ -248,7 +247,7 @@ class SubmitPaste(AbstractModule):
# delete uuid
self.r_serv_db.srem('submitted:uuid', uuid)
self.redis_logger.debug(f'{uuid} all file submitted')
self.logger.debug(f'{uuid} all file submitted')
print(f'{uuid} all file submitted')
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
@ -262,11 +261,11 @@ class SubmitPaste(AbstractModule):
full_path = os.path.join(ITEMS_FOLDER, save_path)
self.redis_logger.debug(f'file path of the paste {full_path}')
self.logger.debug(f'file path of the paste {full_path}')
if not os.path.isfile(full_path):
# file not exists in AIL paste directory
self.redis_logger.debug(f"new paste {paste_content}")
self.logger.debug(f"new paste {paste_content}")
gzip64encoded = self._compress_encode_content(paste_content, uuid)
@ -274,7 +273,7 @@ class SubmitPaste(AbstractModule):
# use relative path
rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1)
self.redis_logger.debug(f"relative path {rel_item_path}")
self.logger.debug(f"relative path {rel_item_path}")
item = Item(rel_item_path)
@ -295,12 +294,12 @@ class SubmitPaste(AbstractModule):
if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'):
self.r_serv_log_submit.set(f'{uuid}:end', 1)
self.redis_logger.debug(f' {rel_item_path} send to Mixer')
self.logger.debug(f' {rel_item_path} send to Mixer')
print(f' {rel_item_path} send to Mixer')
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today()
self.redis_logger.debug("paste submitted")
self.logger.debug("paste submitted")
else:
self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
@ -316,7 +315,7 @@ class SubmitPaste(AbstractModule):
return gzip64encoded
def addError(self, uuid, errorMessage):
self.redis_logger.debug(errorMessage)
self.logger.debug(errorMessage)
print(errorMessage)
error = self.r_serv_log_submit.get(f'{uuid}:error')
if error is not None:
@ -324,7 +323,7 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.incr(f'{uuid}:nb_end')
def abord_file_submission(self, uuid, errorMessage):
self.redis_logger.debug(f'abord {uuid}, {errorMessage}')
self.logger.debug(f'abord {uuid}, {errorMessage}')
self.addError(uuid, errorMessage)
self.r_serv_log_submit.set(f'{uuid}:end', 1)

2
bin/modules/Urls.py
View file

@ -85,8 +85,6 @@ class Urls(AbstractModule):
if len(l_urls) > 0:
to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(to_print)
# .debug ???
# self.redis_logger.info(f'{to_print}Detected {len(l_urls)} URL;{item.get_id()}')
if __name__ == '__main__':

10
bin/modules/abstract_module.py
View file

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from pubsublogger import publisher
from lib import ail_logger
from lib.ail_queues import AILQueue
from lib import regex_helper
@ -51,15 +50,6 @@ class AbstractModule(ABC):
self.obj = None
self.sha256_mess = None
# Init Redis Logger
self.redis_logger = publisher
# Port of the redis instance used by pubsublogger
self.redis_logger.port = 6380
# Channel name to publish logs
# # TODO: refactor logging
# If provided could be a namespaced channel like script:<ModuleName>
self.redis_logger.channel = 'Script'
# Cache key
self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
self.max_execution_time = 30

12
bin/trackers/Retro_Hunt.py
View file

@ -42,12 +42,11 @@ class Retro_Hunt_Module(AbstractModule):
self.obj = None
self.tags = []
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
# # TODO: # start_time
# # end_time
def compute(self, task_uuid):
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
print(f'starting Retro hunt task {task_uuid}')
self.progress = 0
# First launch
@ -58,7 +57,7 @@ class Retro_Hunt_Module(AbstractModule):
timeout = self.retro_hunt.get_timeout()
self.tags = self.retro_hunt.get_tags()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
self.logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
# Filters
filters = self.retro_hunt.get_filters()
@ -111,7 +110,6 @@ class Retro_Hunt_Module(AbstractModule):
# Completed
self.retro_hunt.complete()
print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
def update_progress(self):
if self.nb_objs == 0:
@ -128,7 +126,6 @@ class Retro_Hunt_Module(AbstractModule):
# print(data)
task_uuid = data['namespace']
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id)
@ -158,16 +155,15 @@ class Retro_Hunt_Module(AbstractModule):
task_uuid = Tracker.get_retro_hunt_task_to_start()
if task_uuid:
# Module processing with the message from the queue
self.redis_logger.debug(task_uuid)
self.logger.debug(task_uuid)
# try:
self.compute(task_uuid)
# except Exception as err:
# self.redis_logger.error(f'Error in module {self.module_name}: {err}')
# self.logger.error(f'Error in module {self.module_name}: {err}')
# # Remove uuid ref
# self.remove_submit_uuid(uuid)
else:
# Wait before next process
self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s')
time.sleep(self.pending_seconds)

4
bin/trackers/Tracker_Regex.py
View file

@ -47,14 +47,13 @@ class Tracker_Regex(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked regex
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed')
obj = self.get_obj()
@ -117,7 +116,6 @@ class Tracker_Regex(AbstractModule):
continue
print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

7
bin/trackers/Tracker_Term.py
View file

@ -62,20 +62,18 @@ class Tracker_Term(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked term
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed')
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
obj = self.get_obj()
@ -93,7 +91,7 @@ class Tracker_Term(AbstractModule):
try:
dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException:
self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout")
self.logger.warning(f"{self.obj.get_global_id()} processing timeout")
else:
signal.alarm(0)
@ -125,7 +123,6 @@ class Tracker_Term(AbstractModule):
continue
print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(), obj_id)

4
bin/trackers/Tracker_Typo_Squatting.py
View file

@ -43,14 +43,13 @@ class Tracker_Typo_Squatting(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked typo
if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquatting = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed')
host = message
@ -76,7 +75,6 @@ class Tracker_Typo_Squatting(AbstractModule):
continue
print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

5
bin/trackers/Tracker_Yara.py
View file

@ -44,14 +44,13 @@ class Tracker_Yara(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh YARA list
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
self.obj = self.get_obj()
@ -69,11 +68,9 @@ class Tracker_Yara(AbstractModule):
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}')
print(f'{self.obj.get_global_id()}: {yara_match}')
except yara.TimeoutError:
print(f'{self.obj.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
def convert_byte_offset_to_string(self, b_content, offset):
byte_chunk = b_content[:offset + 1]

3
requirements.txt
View file

@ -25,9 +25,6 @@ xxhash>3.1.0
zmq
pyzmq>19.0.0
# Logging
logbook
pubsublogger
# Tokeniser
nltk>3.4.5

8
var/www/Flask_config.py
View file

@ -16,7 +16,6 @@ import sys
##################################
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
from pubsublogger import publisher
# FLASK #
app = None
@ -34,13 +33,6 @@ r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint
r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint
# Logger (Redis)
redis_logger = publisher
# Port of the redis instance used by pubsublogger
redis_logger.port = 6380
# Channel name to publish logs
redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys')
#### VARIABLES ####

27
var/www/blueprints/ui_submit.py
View file

@ -39,8 +39,6 @@ baseUrl = Flask_config.baseUrl
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
logger = Flask_config.redis_logger
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
@ -57,11 +55,9 @@ def limit_content_length():
def decorator(f):
@wraps(f)
def wrapper(*args, **kwargs):
logger.debug('decorator')
cl = request.content_length
if cl is not None:
if cl > Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE or ('file' not in request.files and cl > Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE):
logger.debug('abort')
abort(413)
return f(*args, **kwargs)
return wrapper
@ -75,7 +71,6 @@ def allowed_file(filename):
return True
else:
file_ext = filename.rsplit('.', 1)[1].lower()
logger.debug(file_ext)
return file_ext in Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
@ -111,7 +106,6 @@ def PasteSubmit_page():
@login_user_no_api
@limit_content_length()
def submit():
logger.debug('submit')
password = request.form['archive_pass']
ltags = request.form['tags_taxonomies']
@ -124,12 +118,10 @@ def submit():
paste_source = paste_source.replace('/', '')[:80]
if paste_source in ['crawled', 'tests']:
content = 'Invalid source'
logger.info(paste_source)
return content, 400
if not re.match('^[0-9a-zA-Z-_\+@#&\.;=:!]*$', paste_source):
content = f'Invalid source name: Forbidden character(s)'
logger.info(content)
return content, 400
is_file = False
@ -139,8 +131,6 @@ def submit():
if file_import.filename:
is_file = True
logger.debug(f'is file ? {is_file}')
submitted_tag = 'infoleak:submission="manual"'
# active taxonomies
@ -149,13 +139,11 @@ def submit():
active_galaxies = Tag.get_active_galaxies()
if ltags or ltagsgalaxies:
logger.debug(f'ltags ? {ltags} {ltagsgalaxies}')
ltags = Tag.unpack_str_tags_list(ltags)
ltagsgalaxies = Tag.unpack_str_tags_list(ltagsgalaxies)
if not Tag.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
content = 'INVALID TAGS'
logger.info(content)
return content, 400
# add submitted tags
@ -164,36 +152,28 @@ def submit():
ltags.append(submitted_tag)
if is_file:
logger.debug('file management')
if allowed_file(file_import.filename):
logger.debug('file extension allowed')
# get UUID
UUID = str(uuid.uuid4())
# create submitted dir
if not os.path.exists(UPLOAD_FOLDER):
logger.debug('create folder')
os.makedirs(UPLOAD_FOLDER)
if '.' not in file_import.filename:
logger.debug('add UUID to path')
full_path = os.path.join(UPLOAD_FOLDER, UUID)
else:
if file_import.filename[-6:] == 'tar.gz':
logger.debug('file extension is tar.gz')
file_type = 'tar.gz'
else:
file_type = file_import.filename.rsplit('.', 1)[1]
logger.debug(f'file type {file_type}')
name = UUID + '.' + file_type
full_path = os.path.join(UPLOAD_FOLDER, name)
logger.debug(f'full path {full_path}')
# Flask verify the file size
file_import.save(full_path)
logger.debug('file saved')
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
@ -204,18 +184,13 @@ def submit():
else:
content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension'
logger.info(content)
return content, 400
elif paste_content != '':
logger.debug(f'entering text paste management')
if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE:
logger.debug(f'size {sys.getsizeof(paste_content)}')
# get id
UUID = str(uuid.uuid4())
logger.debug('create import')
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, source=paste_source)
logger.debug('import OK')
return render_template("submit_items.html",
active_taxonomies = active_taxonomies,
active_galaxies = active_galaxies,
@ -223,11 +198,9 @@ def submit():
else:
content = f'text paste size is over {Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE} bytes limit'
logger.info(content)
return content, 400
content = 'submit aborded'
logger.error(content)
return content, 400
return PasteSubmit_page()