mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
fix: [modules] fix modules
This commit is contained in:
parent
6842efc15d
commit
ab24343b48
10 changed files with 131 additions and 33 deletions
|
@ -217,7 +217,7 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cve.py; read x"
|
||||
screen -S "Script_AIL" -X screen -t "CveModule" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CveModule.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
|
||||
sleep 0.1
|
||||
|
|
|
@ -60,9 +60,6 @@ class MailExporter(AbstractExporter, ABC):
|
|||
self.port is None):
|
||||
raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
|
||||
|
||||
def import(self):
|
||||
pass
|
||||
|
||||
def get_smtp_client(self):
|
||||
# try:
|
||||
if self.pw is not None:
|
||||
|
|
|
@ -26,7 +26,9 @@ from lib.Users import User
|
|||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
|
||||
r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
|
||||
r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
|
||||
|
||||
r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") # TODO REMOVE ME
|
||||
|
||||
items_dir = config_loader.get_config_str("Directories", "pastes")
|
||||
if items_dir[-1] == '/':
|
||||
|
@ -68,12 +70,112 @@ def is_valid_mail(email):
|
|||
def verify_mail_list(mail_list):
|
||||
for mail in mail_list:
|
||||
if not is_valid_mail(mail):
|
||||
return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
|
||||
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
|
||||
return None
|
||||
|
||||
##-- UTILS --##
|
||||
###############
|
||||
|
||||
################################################################################################
|
||||
################################################################################################
|
||||
################################################################################################
|
||||
|
||||
class Tracker:
|
||||
def __init__(self, tracker_uuid):
|
||||
self.uuid = tracker_uuid
|
||||
|
||||
def get_uuid(self):
|
||||
return self.uuid
|
||||
|
||||
def exists(self):
|
||||
return r_tracker.exists(f'tracker:{self.uuid}')
|
||||
|
||||
def get_date(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'date')
|
||||
|
||||
def get_first_seen(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'first_seen')
|
||||
|
||||
def get_last_seen(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'last_seen')
|
||||
|
||||
def get_description(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'description')
|
||||
|
||||
def get_level(self):
|
||||
level = r_tracker.hget(f'tracker:{self.uuid}', 'level')
|
||||
if not level:
|
||||
level = 0
|
||||
return int(level)
|
||||
|
||||
def get_sources(self):
|
||||
return r_tracker.smembers(f'tracker:sources:{self.uuid}')
|
||||
|
||||
def get_tracker(self):
|
||||
return r_serv_tracker.hget(f'tracker:{self.uuid}', 'tracked')
|
||||
|
||||
def get_type(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'type')
|
||||
|
||||
def get_tags(self):
|
||||
return r_tracker.smembers(f'tracker:tags:{self.uuid}')
|
||||
|
||||
def mail_export(self):
|
||||
return r_tracker.exists(f'tracker:mail:{self.uuid}')
|
||||
|
||||
def get_mails(self):
|
||||
return r_tracker.smembers(f'tracker:mail:{self.uuid}')
|
||||
|
||||
def get_user(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'user_id')
|
||||
|
||||
def webhook_export(self):
|
||||
return r_tracker.hexists(f'tracker:mail:{self.uuid}', 'webhook')
|
||||
|
||||
def get_webhook(self):
|
||||
return r_tracker.hget(f'tracker:{self.uuid}', 'webhook')
|
||||
|
||||
# TODO get objects/ tracked items
|
||||
|
||||
|
||||
# TODO sparkline
|
||||
def get_meta(self, options):
|
||||
if not options:
|
||||
options = set()
|
||||
meta = {'uuid': self.uuid,
|
||||
'tracker': self.get_tracker(),
|
||||
'type': self.get_type(),
|
||||
'date': self.get_date(),
|
||||
'first_seen': self.get_first_seen(),
|
||||
'last_seen': self.get_last_seen()}
|
||||
if 'user' in options:
|
||||
meta['user'] = self.get_user()
|
||||
if 'level' in options:
|
||||
meta['level'] = self.get_level()
|
||||
if 'description' in options:
|
||||
meta['description'] = self.get_description()
|
||||
if 'tags' in options:
|
||||
meta['tags'] = self.get_tags()
|
||||
if 'sources' in options:
|
||||
meta['sources'] = self.get_sources()
|
||||
if 'mails' in options:
|
||||
meta['mails'] = self.get_mails()
|
||||
if 'webhooks' in options:
|
||||
meta['webhook'] = self.get_webhook()
|
||||
# if 'sparkline' in options:
|
||||
# meta['sparkline'] = get_tracker_sparkline(tracker_uuid)
|
||||
|
||||
|
||||
|
||||
|
||||
# TODO
|
||||
def add(self, obj_id):
|
||||
pass
|
||||
|
||||
################################################################################################
|
||||
################################################################################################
|
||||
################################################################################################
|
||||
|
||||
def get_all_tracker_type():
|
||||
return ['word', 'set', 'regex', 'yara']
|
||||
|
||||
|
@ -345,7 +447,7 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
|
|||
|
||||
## API ##
|
||||
def api_check_tracker_uuid(tracker_uuid):
|
||||
if not is_valid_uuid_v4(task_uuid):
|
||||
if not is_valid_uuid_v4(tracker_uuid):
|
||||
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
|
@ -678,6 +780,10 @@ def reload_yara_rules():
|
|||
l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
|
||||
for tracker_uuid in l_tracker_uuid:
|
||||
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
|
||||
for tracker_uuid in rule_dict:
|
||||
if not os.path.isfile(rule_dict[tracker_uuid]):
|
||||
# TODO IGNORE + LOGS
|
||||
raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists")
|
||||
rules = yara.compile(filepaths=rule_dict)
|
||||
return rules
|
||||
|
||||
|
|
|
@ -40,7 +40,7 @@ def is_passive_dns_enabled(cache=True):
|
|||
res = r_cache.get('d4:passivedns:enabled')
|
||||
if res is None:
|
||||
res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
|
||||
r_cache.set('d4:passivedns:enabled', res)
|
||||
r_cache.set('d4:passivedns:enabled', str(res))
|
||||
return res
|
||||
else:
|
||||
return res == 'True'
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import magic
|
||||
import requests
|
||||
import zipfile
|
||||
|
||||
|
@ -135,9 +136,9 @@ class Decoded(AbstractDaterangeObject):
|
|||
obj.first_seen = self.get_first_seen()
|
||||
obj.last_seen = self.get_last_seen()
|
||||
|
||||
obj_attrs.append( obj.add_attribute('sha1', value=self.id))
|
||||
obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype()))
|
||||
obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
|
||||
obj_attrs.append(obj.add_attribute('sha1', value=self.id))
|
||||
obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype()))
|
||||
obj_attrs.append(obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
|
||||
for obj_attr in obj_attrs:
|
||||
for tag in self.get_tags():
|
||||
obj_attr.add_tag(tag)
|
||||
|
@ -177,7 +178,10 @@ class Decoded(AbstractDaterangeObject):
|
|||
else:
|
||||
return {}
|
||||
|
||||
# TODO
|
||||
def guess_mimetype(self, bytes_content):
|
||||
# if not bytes_content:
|
||||
# bytes_content = self.get_content()
|
||||
return magic.from_buffer(bytes_content, mime=True)
|
||||
|
||||
# avoid counting the same hash multiple time on the same item
|
||||
|
@ -226,15 +230,15 @@ class Decoded(AbstractDaterangeObject):
|
|||
# -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}')
|
||||
|
||||
# # TODO: ADD items
|
||||
def create(self, content, date, mimetype=None):
|
||||
if not mimetype:
|
||||
mimetype = self.guess_mimetype(content)
|
||||
self.save_file(content, mimetype)
|
||||
|
||||
|
||||
update_decoded_daterange(sha1_string, date_from)
|
||||
if date_from != date_to and date_to:
|
||||
update_decoded_daterange(sha1_string, date_to)
|
||||
# def create(self, content, date, mimetype=None):
|
||||
# if not mimetype:
|
||||
# mimetype = self.guess_mimetype(content)
|
||||
# self.save_file(content, mimetype)
|
||||
#
|
||||
#
|
||||
# update_decoded_daterange(sha1_string, date_from)
|
||||
# if date_from != date_to and date_to:
|
||||
# update_decoded_daterange(sha1_string, date_to)
|
||||
|
||||
#######################################################################################
|
||||
#######################################################################################
|
||||
|
@ -266,6 +270,8 @@ class Decoded(AbstractDaterangeObject):
|
|||
# file in queue
|
||||
elif response_code == -2:
|
||||
report = 'In Queue - Refresh'
|
||||
else:
|
||||
report = 'Error - Unknown VT response'
|
||||
self.set_vt_report(report)
|
||||
print(json_response)
|
||||
print(response_code)
|
||||
|
|
|
@ -39,6 +39,7 @@ class ApiKey(AbstractModule):
|
|||
re.compile(self.re_aws_secret_key)
|
||||
|
||||
# r'=AIza[0-9a-zA-Z-_]{35}' keep equal ????
|
||||
# AIza[0-9A-Za-z\\-_]{35}
|
||||
self.re_google_api_key = r'AIza[0-9a-zA-Z-_]{35}'
|
||||
re.compile(self.re_google_api_key)
|
||||
|
||||
|
@ -80,9 +81,6 @@ class ApiKey(AbstractModule):
|
|||
msg = f'infoleak:automatic-detection="api-key";{item.get_id()}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# Send to duplicate
|
||||
self.send_message_to_queue(item.get_id(), 'Duplicate')
|
||||
|
||||
if r_result:
|
||||
return google_api_key, aws_access_key, aws_secret_key
|
||||
|
||||
|
|
|
@ -76,9 +76,6 @@ class LibInjection(AbstractModule):
|
|||
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
|
||||
self.redis_logger.warning(to_print)
|
||||
|
||||
# Send to duplicate
|
||||
self.send_message_to_queue(item_id, 'Duplicate')
|
||||
|
||||
# Add tag
|
||||
msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
|
|
@ -56,9 +56,6 @@ class Phone(AbstractModule):
|
|||
msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# Send to duplicate
|
||||
self.send_message_to_queue(item.get_id(), 'Duplicate')
|
||||
|
||||
stats = {}
|
||||
for phone_number in results:
|
||||
try:
|
||||
|
|
|
@ -57,9 +57,6 @@ class SQLInjectionDetection(AbstractModule):
|
|||
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
|
||||
self.redis_logger.warning(to_print)
|
||||
|
||||
# Send to duplicate
|
||||
self.send_message_to_queue(item_id, 'Duplicate')
|
||||
|
||||
# Tag
|
||||
msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
|
|
@ -36,7 +36,7 @@ publish = Redis_Host
|
|||
subscribe = Redis_Host
|
||||
publish = Redis_D4_client
|
||||
|
||||
[D4_client]
|
||||
[D4Client]
|
||||
subscribe = Redis_D4_client
|
||||
|
||||
[Retro_Hunt]
|
||||
|
|
Loading…
Reference in a new issue