mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
261 lines
9 KiB
Python
Executable file
261 lines
9 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
"""
|
|
|
|
|
|
"""
|
|
import os
|
|
import sys
|
|
|
|
import importlib.util
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from lib.ConfigLoader import ConfigLoader
|
|
from lib import Users
|
|
|
|
# # # # CONFIGS # # # #
|
|
config_loader = ConfigLoader()
|
|
r_kvrocks = config_loader.get_redis_conn("Kvrocks_DB")
|
|
|
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
|
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
|
config_loader = None
|
|
# # - - CONFIGS - - # #
|
|
|
|
from core import ail_2_ail
|
|
spec = importlib.util.find_spec('ail_2_ail')
|
|
old_ail_2_ail = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(old_ail_2_ail)
|
|
|
|
old_ail_2_ail.r_serv_sync = r_serv_db
|
|
|
|
from lib import Tracker
|
|
spec = importlib.util.find_spec('Tracker')
|
|
old_Tracker = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(old_Tracker)
|
|
|
|
old_Tracker.r_serv_tracker = r_serv_tracker
|
|
|
|
from lib import Investigations
|
|
spec = importlib.util.find_spec('Investigations')
|
|
old_Investigations = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(old_Investigations)
|
|
|
|
old_Investigations.r_tracking = r_serv_tracker
|
|
|
|
|
|
# # TODO: desable features - credentials - stats ? - sentiment analysis
|
|
|
|
# CREATE FUNCTION BY DB/FEATURES
|
|
|
|
# /!\ ISSUE WITH FILE DUPLICATES => NEED TO BE REFACTORED
|
|
|
|
def core_migration():
|
|
print('CORE MIGRATION...')
|
|
|
|
# AIL UUID
|
|
ail_uuid = r_serv_db.get('ail:uuid')
|
|
r_kvrocks.set('ail:uuid', ail_uuid)
|
|
|
|
# AIL update # # TODO: TO TEST
|
|
ail_version = r_serv_db.get('ail:version')
|
|
r_kvrocks.set('ail:version', ail_version)
|
|
dict_update = r_serv_db.hgetall('ail:update_date')
|
|
for version in dict_update:
|
|
r_kvrocks.hset('ail:update_date', version, dict_update[version])
|
|
|
|
versions_to_update = r_serv_db.smembers('ail:to_update')
|
|
for version in versions_to_update:
|
|
r_kvrocks.sadd('ail:update:to_update', version)
|
|
update_error = r_serv_db.get('ail:update_error')
|
|
update_in_progress = r_serv_db.get('ail:update_in_progress')
|
|
r_kvrocks.set('ail:update:error', update_error)
|
|
r_kvrocks.set('ail:update:update_in_progress', update_in_progress)
|
|
|
|
# d4 passivedns
|
|
d4_enabled = r_serv_db.hget('d4:passivedns', 'enabled')
|
|
d4_update_time = r_serv_db.hget('d4:passivedns', 'update_time')
|
|
r_kvrocks.hset('d4:passivedns', 'enabled', bool(d4_enabled))
|
|
r_kvrocks.hset('d4:passivedns', 'update_time', d4_update_time)
|
|
|
|
# ail:misp
|
|
# ail:thehive
|
|
# hive:auto-alerts
|
|
# list_export_tags
|
|
# misp:auto-events
|
|
# whitelist_hive
|
|
# whitelist_misp
|
|
|
|
|
|
# # TODO: TO CHECK
|
|
# config:all_global_section +
|
|
# config:global:crawler +
|
|
# mess_not_saved_export
|
|
|
|
|
|
# # # # # # # # # # # # # # # #
|
|
# USERS
|
|
#
|
|
# HSET 'user:all' user_id passwd_hash
|
|
# HSET 'user:tokens' token user_id
|
|
# HSET 'user_metadata:{user_id}' 'token' token
|
|
# 'role' role
|
|
# 'change_passwd' 'True'
|
|
# SET 'user_role:{role}' user_id
|
|
#
|
|
def user_migration():
|
|
print('USER MIGRATION...')
|
|
|
|
# create role_list
|
|
Users._create_roles_list()
|
|
|
|
for user_id in r_serv_db.hkeys('user:all'):
|
|
role = r_serv_db.hget(f'user_metadata:{user_id}', 'role')
|
|
password_hash = r_serv_db.hget('user:all', user_id)
|
|
token = r_serv_db.hget(f'user_metadata:{user_id}', 'token')
|
|
chg_passwd = r_serv_db.hget(f'user_metadata:{user_id}', 'change_passwd')
|
|
if not chg_passwd:
|
|
chg_passwd = None
|
|
|
|
Users.create_user(user_id, password=None, chg_passwd=chg_passwd, role=role)
|
|
Users.edit_user_password(user_id, password_hash, chg_passwd=chg_passwd)
|
|
Users._delete_user_token(user_id)
|
|
Users._set_user_token(user_id, token)
|
|
|
|
# # # # # # # # # # # # # # # #
|
|
# AIL 2 AIL
|
|
def ail_2_ail_migration():
|
|
print('AIL_2_AIL MIGRATION...')
|
|
|
|
# AIL Queues
|
|
for queue_uuid in old_ail_2_ail.get_all_sync_queue():
|
|
#print(queue_uuid)
|
|
meta = old_ail_2_ail.get_sync_queue_metadata(queue_uuid)
|
|
|
|
name = meta['name']
|
|
tags = meta['tags']
|
|
description = meta['description']
|
|
max_size = meta['max_size']
|
|
ail_2_ail.create_sync_queue(name, tags=tags, description=description, max_size=max_size, _queue_uuid=queue_uuid)
|
|
|
|
# AIL Instances
|
|
for ail_uuid in old_ail_2_ail.get_all_ail_instance():
|
|
#print(ail_uuid)
|
|
meta = old_ail_2_ail.get_ail_instance_metadata(ail_uuid, client_sync_mode=True, server_sync_mode=True, sync_queues=True)
|
|
url = meta['url']
|
|
api_key = meta['api_key']
|
|
description = meta['description']
|
|
pull = meta['pull']
|
|
push = meta['push']
|
|
ail_2_ail.create_ail_instance(ail_uuid, url, api_key=api_key, description=description, pull=pull, push=push)
|
|
|
|
version = old_ail_2_ail.get_ail_server_version(ail_uuid)
|
|
if version:
|
|
ail_2_ail.set_ail_server_version(ail_uuid, version)
|
|
ping = old_ail_2_ail.get_ail_server_ping(ail_uuid)
|
|
if ping:
|
|
ail_2_ail.set_ail_server_ping(ail_uuid, ping)
|
|
error = old_ail_2_ail.get_ail_server_error(ail_uuid)
|
|
if error:
|
|
ail_2_ail.save_ail_server_error(ail_uuid, error)
|
|
|
|
for queue_uuid in meta['sync_queues']:
|
|
ail_2_ail.register_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
|
|
for dict_obj in reversed(old_ail_2_ail.get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=True)):
|
|
ail_2_ail.add_object_to_sync_queue(queue_uuid, ail_uuid, dict_obj, push=True, pull=False, json_obj=False)
|
|
|
|
for dict_obj in reversed(old_ail_2_ail.get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=False)):
|
|
ail_2_ail.add_object_to_sync_queue(queue_uuid, ail_uuid, dict_obj, push=False, pull=True, json_obj=False)
|
|
|
|
# server
|
|
# queue
|
|
# item in queue
|
|
ail_2_ail.set_last_updated_sync_config()
|
|
|
|
# trackers + retro_hunts
|
|
def trackers_migration():
|
|
print('TRACKERS MIGRATION...')
|
|
for tracker_uuid in old_Tracker.get_all_tracker_uuid():
|
|
meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True)
|
|
Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen'])
|
|
|
|
# object migration # # TODO: in background
|
|
for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']):
|
|
Tracker.add_tracked_item(tracker_uuid, item_id)
|
|
|
|
print('RETRO HUNT MIGRATION...')
|
|
|
|
for task_uuid in old_Tracker.get_all_retro_hunt_tasks():
|
|
meta = old_Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True, sources=True, tags=True, description=True, nb_match=True)
|
|
last_id = old_Tracker.get_retro_hunt_last_analyzed(task_uuid)
|
|
timeout = old_Tracker.get_retro_hunt_task_timeout(task_uuid)
|
|
Tracker._re_create_retro_hunt_task(meta['name'], meta['rule'], meta['date'], meta['date_from'], meta['date_to'], meta['creator'], meta['sources'], meta['tags'], [], timeout, meta['description'], task_uuid, state=meta['state'], nb_match=meta['nb_match'], last_id=last_id)
|
|
|
|
# # TODO: IN background ?
|
|
for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']):
|
|
Tracker.save_retro_hunt_match(task_uuid, id)
|
|
|
|
|
|
def investigations_migration():
|
|
print('INVESTIGATION MIGRATION...')
|
|
for investigation_uuid in old_Investigations.get_all_investigations():
|
|
old_investigation = old_Investigations.Investigation(investigation_uuid)
|
|
meta = old_investigation.get_metadata()
|
|
Investigations._re_create_investagation(meta['uuid'], meta['user_creator'], meta['date'], meta['name'], meta['threat_level'], meta['analysis'], meta['info'], meta['tags'], meta['last_change'], meta['timestamp'], meta['misp_events'])
|
|
new_investigation = Investigations.Investigation(meta['uuid'])
|
|
for dict_obj in old_investigation.get_objects():
|
|
new_investigation.register_object(dict_obj['id'], dict_obj['type'], dict_obj['subtype'])
|
|
new_investigation.set_last_change(meta['last_change'])
|
|
|
|
def item_submit_migration():
|
|
pass
|
|
|
|
# /!\ KEY COLISION
|
|
# # TODO: change db
|
|
def tags_migration():
|
|
|
|
pass
|
|
|
|
def items_migration():
|
|
pass
|
|
|
|
def crawler_migration():
|
|
|
|
pass
|
|
|
|
def domain_migration():
|
|
pass
|
|
|
|
# # TODO: refractor keys
|
|
def correlations_migration():
|
|
pass
|
|
|
|
# # # # # # # # # # # # # # # #
|
|
# STATISTICS
|
|
#
|
|
# Credential:
|
|
# HSET 'credential_by_tld:'+date, tld, 1
|
|
def statistics_migration():
|
|
pass
|
|
|
|
if __name__ == '__main__':
|
|
|
|
core_migration()
|
|
user_migration()
|
|
#ail_2_ail_migration()
|
|
trackers_migration()
|
|
#investigations_migration()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
##########################################################
|