From 50f7a31820b348a39271a6ea8698bfe9a40a9e04 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 8 Jul 2022 09:47:47 +0200 Subject: [PATCH] chg: [Kvrocks migration] add users + updates + trackers + retro_hunts + investigation migration --- .gitignore | 2 + bin/DB_KVROCKS_MIGRATION.py | 258 ++++++++++++++++ bin/core/ail_2_ail.py | 31 +- bin/lib/Investigations.py | 32 +- bin/lib/Tracker.py | 64 +++- bin/lib/Users.py | 277 ++++++++++++++++++ bin/lib/ail_updates.py | 183 ++++++++++++ bin/lib/d4.py | 2 +- bin/lib/objects/CryptoCurrencies.py | 65 +++- bin/lib/objects/Decodeds.py | 65 +++- bin/lib/objects/Domains.py | 132 ++++++++- bin/lib/objects/Items.py | 77 ++++- bin/lib/objects/Pgps.py | 18 ++ bin/lib/objects/Screenshots.py | 47 ++- bin/lib/objects/Usernames.py | 37 ++- bin/lib/objects/abstract_object.py | 28 +- bin/lib/objects/abstract_subtype_object.py | 155 ++++++++++ bin/lib/objects/ail_objects.py | 65 +++- bin/modules/Hosts.py | 10 +- bin/packages/git_status.py | 24 ++ bin/update-background.py | 70 ++--- update/bin/ail_updater.py | 27 +- update/bin/old_ail_updater.py | 49 ++++ update/default_update/Update.py | 21 +- update/v3.4/Update.py | 2 +- update/v3.5/Update.py | 2 +- update/v3.6/Update.py | 2 +- update/v3.7/Update.py | 2 +- update/v4.0/Update.py | 2 +- update/v4.1/Update.py | 2 +- update/v4.2/Update.py | 2 +- var/www/Flask_server.py | 8 +- var/www/blueprints/ail_2_ail_sync.py | 1 - var/www/blueprints/hunters.py | 1 - var/www/blueprints/root.py | 17 +- var/www/blueprints/settings_b.py | 66 +++++ var/www/create_default_user.py | 39 +-- var/www/modules/Role_Manager.py | 149 ---------- var/www/modules/dashboard/Flask_dashboard.py | 24 +- .../modules/dashboard/templates/index.html | 7 +- var/www/modules/settings/Flask_settings.py | 159 ++-------- .../settings/templates/header_settings.html | 2 +- var/www/templates/nav_bar.html | 2 +- var/www/templates/settings/menu_sidebar.html | 11 +- .../templates/settings/settings_index.html | 215 ++++++++++++++ 45 files changed, 1925 insertions(+), 529 deletions(-) create mode 100755 bin/DB_KVROCKS_MIGRATION.py create mode 100755 bin/lib/Users.py create mode 100755 bin/lib/ail_updates.py create mode 100755 bin/lib/objects/abstract_subtype_object.py create mode 100755 update/bin/old_ail_updater.py create mode 100644 var/www/blueprints/settings_b.py create mode 100644 var/www/templates/settings/settings_index.html diff --git a/.gitignore b/.gitignore index b12d5a13..0f7dfafa 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ AILENV redis-leveldb redis ardb +kvrocks +DATA_KVROCKS faup tlsh Blooms diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py new file mode 100755 index 00000000..e5f678df --- /dev/null +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" + + +""" +import os +import sys + +import importlib.util + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib import Users + +# # # # CONFIGS # # # # +config_loader = ConfigLoader() +r_kvrocks = config_loader.get_redis_conn("Kvrocks_DB") + +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") +config_loader = None +# # - - CONFIGS - - # # + +from core import ail_2_ail +spec = importlib.util.find_spec('ail_2_ail') +old_ail_2_ail = importlib.util.module_from_spec(spec) +spec.loader.exec_module(old_ail_2_ail) + +old_ail_2_ail.r_serv_sync = r_serv_db + +from lib import Tracker +spec = importlib.util.find_spec('Tracker') +old_Tracker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(old_Tracker) + +old_Tracker.r_serv_tracker = r_serv_tracker + +from lib import Investigations +spec = importlib.util.find_spec('Investigations') +old_Investigations = importlib.util.module_from_spec(spec) +spec.loader.exec_module(old_Investigations) + +old_Investigations.r_tracking = r_serv_tracker + + +# # TODO: desable features - credentials - stats ? - sentiment analysis + +# CREATE FUNCTION BY DB/FEATURES + +# /!\ ISSUE WITH FILE DUPLICATES => NEED TO BE REFACTORED + +def core_migration(): + print('CORE MIGRATION...') + + # AIL UUID + ail_uuid = r_serv_db.get('ail:uuid') + r_kvrocks.set('ail:uuid', ail_uuid) + + # AIL update # # TODO: TO TEST + ail_version = r_serv_db.get('ail:version') + r_kvrocks.set('ail:version', ail_version) + dict_update = r_serv_db.hgetall('ail:update_date') + for version in dict_update: + r_kvrocks.hset('ail:update_date', version, dict_update[version]) + + versions_to_update = r_serv_db.smembers('ail:to_update') + for version in versions_to_update: + r_kvrocks.sadd('ail:update:to_update', version) + update_error = r_serv_db.get('ail:update_error') + update_in_progress = r_serv_db.get('ail:update_in_progress') + r_kvrocks.set('ail:update:error', update_error) + r_kvrocks.set('ail:update:update_in_progress', update_in_progress) + + # d4 passivedns + d4_enabled = r_serv_db.hget('d4:passivedns', 'enabled') + d4_update_time = r_serv_db.hget('d4:passivedns', 'update_time') + r_kvrocks.hset('d4:passivedns', 'enabled', bool(d4_enabled)) + r_kvrocks.hset('d4:passivedns', 'update_time', d4_update_time) + + # ail:misp + # ail:thehive + # hive:auto-alerts + # list_export_tags + # misp:auto-events + # whitelist_hive + # whitelist_misp + + + # # TODO: TO CHECK + # config:all_global_section + + # config:global:crawler + + # mess_not_saved_export + + +# # # # # # # # # # # # # # # # +# USERS +# +# HSET 'user:all' user_id passwd_hash +# HSET 'user:tokens' token user_id +# HSET 'user_metadata:{user_id}' 'token' token +# 'role' role +# 'change_passwd' 'True' +# SET 'user_role:{role}' user_id +# +def user_migration(): + print('USER MIGRATION...') + + # create role_list + Users._create_roles_list() + + for user_id in r_serv_db.hkeys('user:all'): + role = r_serv_db.hget(f'user_metadata:{user_id}', 'role') + password_hash = r_serv_db.hget('user:all', user_id) + token = r_serv_db.hget(f'user_metadata:{user_id}', 'token') + chg_passwd = r_serv_db.hget(f'user_metadata:{user_id}', 'change_passwd') + if not chg_passwd: + chg_passwd = None + + Users.create_user(user_id, password=None, chg_passwd=chg_passwd, role=role) + Users.edit_user_password(user_id, password_hash, chg_passwd=chg_passwd) + Users._delete_user_token(user_id) + Users._set_user_token(user_id, token) + +# # # # # # # # # # # # # # # # +# AIL 2 AIL +def ail_2_ail_migration(): + print('AIL_2_AIL MIGRATION...') + + # AIL Queues + for queue_uuid in old_ail_2_ail.get_all_sync_queue(): + #print(queue_uuid) + meta = old_ail_2_ail.get_sync_queue_metadata(queue_uuid) + + name = meta['name'] + tags = meta['tags'] + description = meta['description'] + max_size = meta['max_size'] + ail_2_ail.create_sync_queue(name, tags=tags, description=description, max_size=max_size, _queue_uuid=queue_uuid) + + # AIL Instances + for ail_uuid in old_ail_2_ail.get_all_ail_instance(): + #print(ail_uuid) + meta = old_ail_2_ail.get_ail_instance_metadata(ail_uuid, client_sync_mode=True, server_sync_mode=True, sync_queues=True) + url = meta['url'] + api_key = meta['api_key'] + description = meta['description'] + pull = meta['pull'] + push = meta['push'] + ail_2_ail.create_ail_instance(ail_uuid, url, api_key=api_key, description=description, pull=pull, push=push) + + version = old_ail_2_ail.get_ail_server_version(ail_uuid) + if version: + ail_2_ail.set_ail_server_version(ail_uuid, version) + ping = old_ail_2_ail.get_ail_server_ping(ail_uuid) + if ping: + ail_2_ail.set_ail_server_ping(ail_uuid, ping) + error = old_ail_2_ail.get_ail_server_error(ail_uuid) + if error: + ail_2_ail.save_ail_server_error(ail_uuid, error) + + for queue_uuid in meta['sync_queues']: + ail_2_ail.register_ail_to_sync_queue(ail_uuid, queue_uuid) + + for dict_obj in reversed(old_ail_2_ail.get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=True)): + ail_2_ail.add_object_to_sync_queue(queue_uuid, ail_uuid, dict_obj, push=True, pull=False, json_obj=False) + + for dict_obj in reversed(old_ail_2_ail.get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=False)): + ail_2_ail.add_object_to_sync_queue(queue_uuid, ail_uuid, dict_obj, push=False, pull=True, json_obj=False) + + # server + # queue + # item in queue + ail_2_ail.set_last_updated_sync_config() + +# trackers + retro_hunts +def trackers_migration(): + print('TRACKERS MIGRATION...') + for tracker_uuid in old_Tracker.get_all_tracker_uuid(): + meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True) + Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen']) + + # object migration # # TODO: in background + for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']): + Tracker.add_tracked_item(tracker_uuid, item_id) + + print('RETRO HUNT MIGRATION...') + + for task_uuid in old_Tracker.get_all_retro_hunt_tasks(): + meta = old_Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True, sources=True, tags=True, description=True, nb_match=True) + last_id = old_Tracker.get_retro_hunt_last_analyzed(task_uuid) + timeout = old_Tracker.get_retro_hunt_task_timeout(task_uuid) + Tracker._re_create_retro_hunt_task(meta['name'], meta['rule'], meta['date'], meta['date_from'], meta['date_to'], meta['creator'], meta['sources'], meta['tags'], [], timeout, meta['description'], task_uuid, state=meta['state'], nb_match=meta['nb_match'], last_id=last_id) + + # # TODO: IN background ? + for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']): + Tracker.save_retro_hunt_match(task_uuid, id) + + +def investigations_migration(): + print('INVESTIGATION MIGRATION...') + for investigation_uuid in old_Investigations.get_all_investigations(): + old_investigation = old_Investigations.Investigation(investigation_uuid) + meta = old_investigation.get_metadata() + Investigations._re_create_investagation(meta['uuid'], meta['user_creator'], meta['date'], meta['name'], meta['threat_level'], meta['analysis'], meta['info'], meta['tags'], meta['last_change'], meta['timestamp'], meta['misp_events']) + new_investigation = Investigations.Investigation(meta['uuid']) + for dict_obj in old_investigation.get_objects(): + new_investigation.register_object(dict_obj['id'], dict_obj['type'], dict_obj['subtype']) + new_investigation.set_last_change(meta['last_change']) + +def item_submit_migration(): + pass + +# /!\ KEY COLISION +# # TODO: change db +def tags_migration(): + + pass + +def crawler_migration(): + + pass + +def domain_migration(): + pass + +# # TODO: refractor keys +def correlations_migration(): + pass + +# # # # # # # # # # # # # # # # +# STATISTICS +# +# Credential: +# HSET 'credential_by_tld:'+date, tld, 1 +def statistics_migration(): + pass + +if __name__ == '__main__': + + core_migration() + user_migration() + #ail_2_ail_migration() + #trackers_migration() + #investigations_migration() + + + + + + + + + + ########################################################## diff --git a/bin/core/ail_2_ail.py b/bin/core/ail_2_ail.py index 2dd12ea6..48f9fd3a 100755 --- a/bin/core/ail_2_ail.py +++ b/bin/core/ail_2_ail.py @@ -26,8 +26,8 @@ import Tag config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") -r_serv_db = config_loader.get_redis_conn("ARDB_DB") -r_serv_sync = config_loader.get_redis_conn("ARDB_DB") +r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") +r_serv_sync = config_loader.get_redis_conn("Kvrocks_DB") config_loader = None WEBSOCKETS_CLOSE_CODES = { @@ -480,7 +480,7 @@ def change_pull_push_state(ail_uuid, pull=None, push=None): else: pull = False if curr_pull != pull: - print('pull hset') + #print('pull hset') r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'pull', pull) edited = True if push is not None: @@ -490,7 +490,7 @@ def change_pull_push_state(ail_uuid, pull=None, push=None): else: push = False if curr_push != push: - print('push hset') + #print('push hset') r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'push', push) edited = True if edited: @@ -991,8 +991,12 @@ def edit_sync_queue_filter_tags(queue_uuid, new_tags): # # TODO: optionnal name ??? # # TODO: SANITYZE TAGS -def create_sync_queue(name, tags=[], description=None, max_size=100): - queue_uuid = generate_uuid() +# # TODO: SANITYZE queue_uuid +def create_sync_queue(name, tags=[], description=None, max_size=100, _queue_uuid=None): + if _queue_uuid: + queue_uuid = sanityze_uuid(_queue_uuid).replace('-', '') + else: + queue_uuid = generate_uuid() r_serv_sync.sadd('ail2ail:sync_queue:all', queue_uuid) r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'name', name) @@ -1156,8 +1160,19 @@ def get_sync_queue_object_by_queue_uuid(queue_uuid, ail_uuid, push=True): # # REVIEW: # TODO: create by obj type return Item(obj_dict['id']) -def add_object_to_sync_queue(queue_uuid, ail_uuid, obj_dict, push=True, pull=True): - obj = json.dumps(obj_dict) +def get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=True): + if push: + sync_mode = 'push' + else: + sync_mode = 'pull' + return r_serv_sync.lrange(f'sync:queue:{sync_mode}:{queue_uuid}:{ail_uuid}', 0, -1) + +# # TODO: use queue max_size +def add_object_to_sync_queue(queue_uuid, ail_uuid, obj_dict, push=True, pull=True, json_obj=True): + if json_obj: + obj = json.dumps(obj_dict) + else: + obj = obj_dict # # TODO: # FIXME: USE CACHE ?????? if push: diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 39a27650..fdf70864 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -28,7 +28,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Tag config_loader = ConfigLoader.ConfigLoader() -r_tracking = config_loader.get_redis_conn("DB_Tracking") +r_tracking = config_loader.get_redis_conn("Kvrocks_DB") config_loader = None @@ -287,12 +287,31 @@ def delete_obj_investigations(obj_id, obj_type, subtype=''): return unregistred +def _set_timestamp(investigation_uuid, timestamp): + r_tracking.hset(f'investigations:data:{investigation_uuid}', 'timestamp', timestamp) + +# analysis - threat level - info - date - creator + +def _re_create_investagation(investigation_uuid, user_id, date, name, threat_level, analysis, info, tags, last_change, timestamp, misp_events): + create_investigation(user_id, date, name, threat_level, analysis, info, tags=tags, investigation_uuid=investigation_uuid) + if timestamp: + _set_timestamp(investigation_uuid, timestamp) + investigation = Investigation(investigation_uuid) + if last_change: + investigation.set_last_change(last_change) + for misp_event in misp_events: + investigation.add_misp_events(misp_event) + # # TODO: fix default threat_level analysis # # TODO: limit description + name # # TODO: sanityze tags # # TODO: sanityze date -def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[]): - investigation_uuid = generate_uuid() +def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[], investigation_uuid=None): + if investigation_uuid: + if not is_valid_uuid_v4(investigation_uuid): + investigation_uuid = generate_uuid() + else: + investigation_uuid = generate_uuid() r_tracking.sadd('investigations:all', investigation_uuid) # user map r_tracking.sadd(f'investigations:user:{user_id}', investigation_uuid) @@ -461,9 +480,10 @@ if __name__ == '__main__': # res = r_tracking.dbsize() # print(res) - investigation_uuid = 'e4e1c8e3b0a349bf81482f2f823efc0f' + investigation_uuid = 'a6545c38083444eeb9383d357f8fa747' + _set_timestamp(investigation_uuid, int(time.time())) - investigation = Investigation(investigation_uuid) - investigation.delete() + # investigation = Investigation(investigation_uuid) + # investigation.delete() # # TODO: PAGINATION diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 4e57b0aa..589581c3 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -25,8 +25,8 @@ import item_basic config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") -r_serv_db = config_loader.get_redis_conn("ARDB_DB") -r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") +r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") +r_serv_tracker = config_loader.get_redis_conn("Kvrocks_DB") items_dir = config_loader.get_config_str("Directories", "pastes") if items_dir[-1] == '/': @@ -99,7 +99,10 @@ def get_tracker_type(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'type') def get_tracker_level(tracker_uuid): - return int(r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'level')) + level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level') + if not level: + level = 0 + return int(level) def get_tracker_user_id(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') @@ -233,6 +236,9 @@ def set_tracker_first_seen(tracker_uuid, date): def set_tracker_last_seen(tracker_uuid, date): r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date)) +def set_tracker_user_id(tracker_uuid, user_id): + r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'user_id', user_id) + # # TODO: ADD CACHE ??? def update_tracker_daterange(tracker_uuid, date, op='add'): date = int(date) @@ -420,6 +426,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): return {"status": "error", "reason": "Incorrect type"}, 400 return {"status": "success", "tracker": tracker, "type": tracker_type}, 200 +def _re_create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard, tracker_uuid, sources, first_seen, last_seen): + create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=dashboard, tracker_uuid=tracker_uuid, sources=sources) + set_tracker_user_id(tracker_uuid, user_id) + if first_seen: + set_tracker_first_seen(tracker_uuid, first_seen) + if last_seen: + set_tracker_last_seen(tracker_uuid, last_seen) + + def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]): # edit tracker if tracker_uuid: @@ -836,8 +851,11 @@ def set_retro_hunt_task_state(task_uuid, new_state): r_serv_tracker.sadd(f'tracker:retro_hunt:task:{new_state}', task_uuid) r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'state', new_state) -def get_retro_hunt_task_type(task_uuid): - return r_serv_tracker(f'tracker:retro_hunt:task:{task_uuid}', 'type') +# def get_retro_hunt_task_type(task_uuid): +# return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'type') +# +# def set_retro_hunt_task_type(task_uuid, task_type): +# r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'type', task_type) # # TODO: yararule def get_retro_hunt_task_rule(task_uuid, r_compile=False): @@ -857,6 +875,12 @@ def get_retro_hunt_task_timeout(task_uuid): else: return 30 # # TODO: FIXME use instance limit +def get_retro_hunt_task_date(task_uuid): + return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date') + +def set_retro_hunt_task_date(task_uuid, date): + return r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date', date) + def get_retro_hunt_task_date_from(task_uuid): return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date_from') @@ -944,8 +968,17 @@ def clear_retro_hunt_task_cache(task_uuid): # name # description +# state error +def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None): + create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, mails=mails, timeout=timeout, description=description, task_uuid=task_uuid, state=state) + if last_id: + set_retro_hunt_last_analyzed(task_uuid, last_id) + _set_retro_hunt_nb_match(task_uuid, nb_match) + set_retro_hunt_task_date(task_uuid, date) + + # # # TODO: TYPE -def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[], tags=[], mails=[], timeout=30, description=None, task_uuid=None): +def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[], tags=[], mails=[], timeout=30, description=None, task_uuid=None, state='pending'): if not task_uuid: task_uuid = str(uuid.uuid4()) @@ -970,9 +1003,11 @@ def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[], r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail)) r_serv_tracker.sadd('tracker:retro_hunt:task:all', task_uuid) + # add to pending tasks - r_serv_tracker.sadd('tracker:retro_hunt:task:pending', task_uuid) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'state', 'pending') + if state not in ('pending', 'completed', 'paused'): + state = 'pending' + set_retro_hunt_task_state(task_uuid, state) return task_uuid # # TODO: delete rule @@ -1092,12 +1127,15 @@ def compute_retro_hunt_task_progress(task_uuid, date_from=None, date_to=None, so # # TODO: # FIXME: # Cache +# # TODO: ADD MAP ID => Retro_Hunt def save_retro_hunt_match(task_uuid, id, object_type='item'): item_date = item_basic.get_item_date(id) res = r_serv_tracker.sadd(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}', id) # track nb item by date if res == 1: r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1) + # Add map obj_id -> task_uuid + r_serv_tracker.sadd(f'obj:retro_hunt:item:{item_id}', task_uuid) def get_retro_hunt_all_item_dates(task_uuid): return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1) @@ -1111,12 +1149,15 @@ def get_retro_hunt_nb_match(task_uuid): nb_match += int(tuple[1]) return int(nb_match) +def _set_retro_hunt_nb_match(task_uuid, nb_match): + r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match', nb_match) + def set_retro_hunt_nb_match(task_uuid): l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True) nb_match = 0 for tuple in l_date_value: nb_match += int(tuple[1]) - r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match', nb_match) + _set_retro_hunt_nb_match(task_uuid, nb_match) def get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to): all_item_id = set() @@ -1296,8 +1337,9 @@ def _fix_db_custom_tags(): #### -- #### -if __name__ == '__main__': - _fix_db_custom_tags() +#if __name__ == '__main__': + + #_fix_db_custom_tags() # fix_all_tracker_uuid_list() # res = get_all_tracker_uuid() # print(len(res)) diff --git a/bin/lib/Users.py b/bin/lib/Users.py new file mode 100755 index 00000000..15efb6a6 --- /dev/null +++ b/bin/lib/Users.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import bcrypt +import os +import re +import secrets +import sys + +from flask_login import UserMixin + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +# Config +config_loader = ConfigLoader() +#r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") +config_loader = None + +regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,100}$' +regex_password = re.compile(regex_password) + +# # TODO: ADD FUNCTIONS PASSWORD RESET + API KEY RESET + CREATE USER + +# # TODO: migrate Role_Manager + +#### PASSWORDS + TOKENS #### + +def check_password_strength(password): + result = regex_password.match(password) + if result: + return True + else: + return False + +def gen_password(): + return secrets.token_urlsafe(30) + +def hashing_password(password): + password = password.encode() + return bcrypt.hashpw(password, bcrypt.gensalt()) + +def gen_token(): + return secrets.token_urlsafe(41) + +def _delete_user_token(user_id): + current_token = get_user_token(user_id) + r_serv_db.hdel('ail:users:tokens', current_token) + +def _set_user_token(user_id, token): + r_serv_db.hset('ail:users:tokens', token, user_id) + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'token', token) + +def generate_new_token(user_id): + # create user token + _delete_user_token(user_id) + token = gen_token() + _set_user_token(user_id, token) + +def get_default_admin_token(): + if r_serv_db.exists('ail:user:metadata:admin@admin.test'): + return r_serv_db.hget('ail:user:metadata:admin@admin.test', 'token') + else: + return '' + +##-- PASSWORDS + TOKENS --## + +#### USERS #### + +def get_all_users(): + return r_serv_db.hkeys('ail:users:all') + +def get_user_role(user_id): + return r_serv_db.hget(f'ail:user:metadata:{user_id}', 'role') + +def get_user_passwd_hash(user_id): + return r_serv_db.hget('ail:users:all', user_id) + +def get_user_token(user_id): + return r_serv_db.hget(f'ail:users:metadata:{user_id}', 'token') + +def exists_user(user_id): + return r_serv_db.exists(f'ail:user:metadata:{user_id}') + +def get_user_metadata(user_id): + user_metadata = {} + user_metadata['email'] = user_id + user_metadata['role'] = r_serv_db.hget(f'ail:user:metadata:{user_id}', 'role') + user_metadata['api_key'] = r_serv_db.hget(f'ail:user:metadata:{user_id}', 'token') + return user_metadata + +def get_users_metadata(list_users): + users = [] + for user in list_users: + users.append(get_user_metadata(user)) + return users + +def create_user(user_id, password=None, chg_passwd=True, role=None): + # # TODO: check password strenght + if password: + new_password = password + else: + new_password = gen_password() + password_hash = hashing_password(new_password) + + # EDIT + if exists_user(user_id): + if password or chg_passwd: + edit_user_password(user_id, password_hash, chg_passwd=chg_passwd) + if role: + edit_user_role(user_id, role) + # CREATE USER + else: + # Role + if not role: + role = get_default_role() + + if role in get_all_roles(): + for role_to_add in get_all_user_role(role): + r_serv_db.sadd(f'ail:users:role:{role_to_add}', user_id) + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'role', role) + + r_serv_db.hset('ail:users:all', user_id, password_hash) + if chg_passwd: + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'change_passwd', True) + + # create user token + generate_new_token(user_id) + +def edit_user_password(user_id, password_hash, chg_passwd=False): + if chg_passwd: + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'change_passwd', True) + else: + r_serv_db.hdel(f'ail:user:metadata:{user_id}', 'change_passwd') + # remove default user password file + if user_id=='admin@admin.test': + default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') + if os.path.isfile(default_passwd_file): + os.remove(default_passwd_file) + r_serv_db.hset('ail:users:all', user_id, password_hash) + # create new token + generate_new_token(user_id) + +# # TODO: solve edge_case self delete +def delete_user(user_id): + if exists_user(user_id): + for role_id in get_all_roles(): + r_serv_db.srem('ail:users:role:{role_id}', user_id) + user_token = get_user_token(user_id) + r_serv_db.hdel('ail:users:tokens', user_token) + r_serv_db.delete(f'ail:user:metadata:{user_id}') + r_serv_db.hdel('ail:users:all', user_id) + + # # TODO: raise Exception + else: + print(f'Error: user {user_id} do not exist') + +##-- USERS --## + +#### ROLES #### + +def get_all_roles(): + return r_serv_db.zrange('ail:roles:all', 0, -1) + +# create role_list +def _create_roles_list(): + if not r_serv_db.exists('ail:roles:all'): + r_serv_db.zadd('ail:roles:all', 1, 'admin') + r_serv_db.zadd('ail:roles:all', 2, 'analyst') + r_serv_db.zadd('ail:roles:all', 3, 'user') + r_serv_db.zadd('ail:roles:all', 4, 'user_no_api') + r_serv_db.zadd('ail:roles:all', 5, 'read_only') + +def get_role_level(role): + return int(r_serv_db.zscore('ail:roles:all', role)) + +def get_user_role_by_range(inf, sup): + return r_serv_db.zrange('ail:roles:all', inf, sup) + +def get_all_user_role(user_role): + current_role_val = get_role_level(user_role) + return r_serv_db.zrange('ail:roles:all', current_role_val -1, -1) + +def get_all_user_upper_role(user_role): + current_role_val = get_role_level(user_role) + # remove one rank + if current_role_val > 1: + return r_serv_db.zrange('ail:roles:all', 0, current_role_val -2) + else: + return [] + +def get_default_role(): + return 'read_only' + +def edit_user_role(user_id, role): + current_role = get_user_role(user_id) + if role != current_role: + request_level = get_role_level(role) + current_role = get_role_level(current_role) + + if current_role < request_level: + role_to_remove = get_user_role_by_range(current_role -1, request_level - 2) + for role_id in role_to_remove: + r_serv_db.srem(f'ail:users:role:{role_id}', user_id) + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'role', role) + else: + role_to_add = get_user_role_by_range(request_level -1, current_role) + for role_id in role_to_add: + r_serv_db.sadd(f'ail:users:role:{role_id}', user_id) + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'role', role) + +def check_user_role_integrity(user_id): + user_role = get_user_role(user_id) + all_user_role = get_all_user_role(user_role) + res = True + for role in all_user_role: + if not r_serv_db.sismember(f'ail:users:role:{role}', user_id): + res = False + upper_role = get_all_user_upper_role(user_role) + for role in upper_role: + if r_serv_db.sismember(f'ail:users:role:{role}', user_id): + res = False + return res + +##-- ROLES --## + +class User(UserMixin): + + def __init__(self, id): + + if r_serv_db.hexists('ail:users:all', id): + self.id = id + else: + self.id = "__anonymous__" + + # return True or False + #def is_authenticated(): + + # return True or False + #def is_anonymous(): + + @classmethod + def get(self_class, id): + return self_class(id) + + def user_is_anonymous(self): + if self.id == "__anonymous__": + return True + else: + return False + + def check_password(self, password): + if self.user_is_anonymous(): + return False + + password = password.encode() + hashed_password = r_serv_db.hget('ail:users:all', self.id).encode() + if bcrypt.checkpw(password, hashed_password): + return True + else: + return False + + def request_password_change(self): + if r_serv_db.hget(f'ail:user:metadata:{self.id}', 'change_passwd') == 'True': + return True + else: + return False + + def is_in_role(self, role): + if r_serv_db.sismember(f'ail:users:role:{role}', self.id): + return True + else: + return False diff --git a/bin/lib/ail_updates.py b/bin/lib/ail_updates.py new file mode 100755 index 00000000..0f2bb568 --- /dev/null +++ b/bin/lib/ail_updates.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import datetime +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib import Users + +config_loader = ConfigLoader() +#r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") +r_serv = config_loader.get_redis_conn("ARDB_DB") +config_loader = None + +BACKGROUND_UPDATES = { + 'v1.5':{ + 'nb_updates': 5, + 'message': 'Tags and Screenshots' + }, + 'v2.4':{ + 'nb_updates': 1, + 'message': ' Domains Tags and Correlations' + }, + 'v2.6':{ + 'nb_updates': 1, + 'message': 'Domains Tags and Correlations' + }, + 'v2.7':{ + 'nb_updates': 1, + 'message': 'Domains Tags' + }, + 'v3.4':{ + 'nb_updates': 1, + 'message': 'Domains Languages' + }, + 'v3.7':{ + 'nb_updates': 1, + 'message': 'Trackers first_seen/last_seen' + } +} + +def get_ail_version(): + return r_serv.get('ail:version') + +def get_ail_float_version(): + version = get_ail_version() + if version: + version = float(version[1:]) + else: + version = 0 + return version + +def get_ail_all_updates(date_separator='-'): + dict_update = r_serv.hgetall('ail:update_date') + if date_separator: + for version in dict_update: + u_date = dict_update[version] + dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}' + return dict_update + +def add_ail_update(version): + #Add new AIL version + r_serv.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d")) + #Set current ail version + if float(version[1:]) > get_ail_float_version(): + r_serv.set('ail:version', version) + +def check_version(version): + if version[0] != 'v': + return False + try: + int(version[1]) + int(version[-1]) + int(version[1:].replace('.', '')) + except: + return False + if '..' in version: + return False + return True + +#### UPDATE BACKGROUND #### + +def exits_background_update_to_launch(): + return r_serv.scard('ail:update:to_update') != 0 + +def is_version_in_background_update(version): + return r_serv.sismember('ail:update:to_update', version) + +def get_all_background_updates_to_launch(): + return r_serv.smembers('ail:update:to_update') + +def get_current_background_update(): + return r_serv.get('ail:update:update_in_progress') + +def get_current_background_update_script(): + return r_serv.get('ail:update:current_background_script') + +def get_current_background_update_script_path(version, script_name): + return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name) + +def get_current_background_nb_update_completed(): + return r_serv.scard('ail:update:update_in_progress:completed') + +def get_current_background_update_progress(): + progress = r_serv.get('ail:update:current_background_script_stat') + if not progress: + progress = 0 + return int(progress) + +def get_background_update_error(): + return r_serv.get('ail:update:error') + +def add_background_updates_to_launch(version): + return r_serv.sadd('ail:update:to_update', version) + +def start_background_update(version): + r_serv.delete('ail:update:error') + r_serv.set('ail:update:update_in_progress', version) + +def set_current_background_update_script(script_name): + r_serv.set('ail:update:current_background_script', script_name) + r_serv.set('ail:update:current_background_script_stat', 0) + +def set_current_background_update_progress(progress): + r_serv.set('ail:update:current_background_script_stat', progress) + +def set_background_update_error(error): + r_serv.set('ail:update:error', error) + +def end_background_update_script(): + r_serv.sadd('ail:update:update_in_progress:completed') + +def end_background_update(version): + r_serv.delete('ail:update:update_in_progress') + r_serv.delete('ail:update:current_background_script') + r_serv.delete('ail:update:current_background_script_stat') + r_serv.delete('ail:update:update_in_progress:completed') + r_serv.srem('ail:update:to_update', version) + +def clear_background_update(): + r_serv.delete('ail:update:error') + r_serv.delete('ail:update:update_in_progress') + r_serv.delete('ail:update:current_background_script') + r_serv.delete('ail:update:current_background_script_stat') + r_serv.delete('ail:update:update_in_progress:completed') + +def get_update_background_message(version): + return BACKGROUND_UPDATES[version]['message'] + +# TODO: Detect error in subprocess +def get_update_background_metadata(): + dict_update = {} + version = get_current_background_update() + if version: + dict_update['version'] = version + dict_update['script'] = get_current_background_update_script() + dict_update['script_progress'] = get_current_background_update_progress() + dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates'] + dict_update['nb_completed'] = get_current_background_nb_update_completed() + dict_update['progress'] = int(dict_update['nb_completed']*100/dict_update['nb_update']) + dict_update['error'] = get_background_update_error() + return dict_update + + +##-- UPDATE BACKGROUND --## + + + + + + + + +########################## + +if __name__ == '__main__': + res = check_version('v3.1..1') + print(res) diff --git a/bin/lib/d4.py b/bin/lib/d4.py index e3305f75..20d496ec 100755 --- a/bin/lib/d4.py +++ b/bin/lib/d4.py @@ -11,7 +11,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import ConfigLoader config_loader = ConfigLoader.ConfigLoader() -r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") r_cache = config_loader.get_redis_conn("Redis_Cache") config_loader = None diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index a2cb3339..65dce40c 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -5,24 +5,17 @@ import os import sys import redis -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -from abstract_object import AbstractObject from flask import url_for -config_loader = ConfigLoader.ConfigLoader() +sys.path.append(os.environ['AIL_BIN']) +from lib.ConfigLoader import ConfigLoader +from lib.objects import abstract_object + +config_loader = ConfigLoader() config_loader = None - -################################################################################ -################################################################################ -################################################################################ - -class CryptoCurrency(AbstractObject): +class CryptoCurrency(abstract_object.AbstractObject): """ AIL CryptoCurrency Object. (strings) """ @@ -40,6 +33,23 @@ class CryptoCurrency(AbstractObject): # # TODO: pass + def get_currency_symbol(self): + if self.subtype=='bitcoin': + return 'BTC' + elif self.subtype=='ethereum': + return 'ETH' + elif self.subtype=='bitcoin-cash': + return 'BCH' + elif self.subtype=='litecoin': + return 'LTC' + elif self.subtype=='monero': + return 'XMR' + elif self.subtype=='zcash': + return 'ZEC' + elif self.subtype=='dash': + return 'DASH' + return None + def get_link(self, flask_context=False): if flask_context: url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id) @@ -62,6 +72,22 @@ class CryptoCurrency(AbstractObject): icon = '\uf51e' return {'style': style, 'icon': icon, 'color': '#DDCC77', 'radius':5} + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('coin-address') + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + + obj_attrs.append( obj.add_attribute('address', value=self.id) ) + crypto_symbol = self.get_currency_symbol() + if crypto_symbol: + obj_attrs.append( obj.add_attribute('symbol', value=crypto_symbol) ) + + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + ############################################################################ ############################################################################ ############################################################################ @@ -72,6 +98,19 @@ class CryptoCurrency(AbstractObject): ############################################################################ ############################################################################ +def build_crypto_regex(subtype, search_id): + pass + +def search_by_name(subtype, search_id): + + # # TODO: BUILD regex + obj = CryptoCurrency(subtype, search_id) + if obj.exists(): + return search_id + else: + regex = build_crypto_regex(subtype, search_id) + return abstract_object.search_subtype_obj_by_id('cryptocurrency', subtype, regex) + #if __name__ == '__main__': diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 8f3f82f1..8f4efd01 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -3,18 +3,16 @@ import os import sys -import redis -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -from abstract_object import AbstractObject from flask import url_for +from io import BytesIO -config_loader = ConfigLoader.ConfigLoader() -r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +sys.path.append(os.environ['AIL_BIN']) +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_object import AbstractObject + +config_loader = ConfigLoader() +r_metadata = config_loader.get_redis_conn("ARDB_Metadata") HASH_DIR = config_loader.get_config_str('Directories', 'hash') baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -43,7 +41,7 @@ class Decoded(AbstractObject): return self.id.split('/')[0] def get_file_type(self): - return r_serv_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type') + return r_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type') # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ def delete(self): @@ -71,6 +69,42 @@ class Decoded(AbstractObject): icon = '\uf249' return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5} + ''' + Return the estimed type of a given decoded item. + + :param sha1_string: sha1_string + ''' + def get_estimated_type(self): + return r_metadata.hget(f'metadata_hash:{self.id}', 'estimated_type') + + def get_rel_path(self, mimetype=None): + if not mimetype: + mimetype = self.get_estimated_type() + return os.path.join(HASH_DIR, mimetype, self.id[0:2], self.id) + + def get_filepath(self, mimetype=None): + return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype)) + + def get_file_content(self, mimetype=None): + filepath = self.get_filepath(mimetype=mimetype) + with open(filepath, 'rb') as f: + file_content = BytesIO(f.read()) + return file_content + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('file') + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + + obj_attrs.append( obj.add_attribute('sha1', value=self.id) ) + obj_attrs.append( obj.add_attribute('mimetype', value=self.get_estimated_type()) ) + obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_file_content()) ) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + ############################################################################ ############################################################################ ############################################################################ @@ -78,6 +112,17 @@ class Decoded(AbstractObject): def exist_correlation(self): pass + def create(self, content, date): + + + + + Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype) + ####correlation Decoded.save_item_relationship(sha1_string, item_id) + Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name) + + + ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Domains.py b/bin/lib/objects/Domains.py index 9f82c6c8..28d59787 100755 --- a/bin/lib/objects/Domains.py +++ b/bin/lib/objects/Domains.py @@ -3,18 +3,18 @@ import os import sys -import redis +import time -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -from abstract_object import AbstractObject from flask import url_for -config_loader = ConfigLoader.ConfigLoader() +sys.path.append(os.environ['AIL_BIN']) +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_object import AbstractObject +from lib.item_basic import get_item_children, get_item_date, get_item_url + +config_loader = ConfigLoader() +r_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None @@ -27,8 +27,10 @@ class Domain(AbstractObject): AIL Decoded Object. (strings) """ + # id: domain name def __init__(self, id): super(Domain, self).__init__('domain', id) + self.domain_type = self.get_domain_type() # def get_ail_2_ail_payload(self): # payload = {'raw': self.get_gzip_content(b64=True), @@ -41,6 +43,57 @@ class Domain(AbstractObject): else: return 'regular' + def get_first_seen(selfr_int=False, separator=True): + first_seen = r_onion.hget(f'{self.domain_type}_metadata:{self.id}', 'first_seen') + if first_seen: + if separator: + first_seen = f'{first_seen[0:4]}/{first_seen[4:6]}/{first_seen[6:8]}' + first_seen = int(first_seen) + elif r_int==True: + first_seen = int(first_seen) + return first_seen + + def get_last_check(self, r_int=False, separator=True): + last_check = r_onion.hget(f'{self.domain_type}_metadata:{self.id}', 'last_check') + if last_check is not None: + if separator: + last_check = f'{last_check[0:4]}/{last_check[4:6]}/{last_check[6:8]}' + elif r_format=="int": + last_check = int(last_check) + return last_check + + def get_ports(self): + l_ports = r_onion.hget(f'{self.domain_type}_metadata:{self.id}', 'ports') + if l_ports: + return l_ports.split(";") + return [] + + def get_history_by_port(self, port, status=False, root=False): + ''' + Return . + + :return: + :rtype: list of tuple (item_core, epoch) + ''' + history_tuple = r_onion.zrange(f'crawler_history_{self.domain_type}:{self.id}:{port}', 0, -1, withscores=True) + history = [] + for root_id, epoch in history_tuple: + dict_history = {} + epoch = int(epoch) # force int + dict_history["epoch"] = epoch + dict_history["date"] = time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)) + try: + int(root_item) + if status: + dict_history['status'] = False + except ValueError: + if status: + dict_history['status'] = True + if root: + dict_history['root'] = root_id + history.append(dict_history) + return history + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ def delete(self): # # TODO: @@ -63,6 +116,69 @@ class Domain(AbstractObject): icon = '\uf13b' return {'style': style, 'icon': icon, 'color':color, 'radius':5} + def is_crawled_item(self, item_id): + domain_lenght = len(self.id) + if len(item_id) > (domain_lenght+48): + if item_id[-36-domain_lenght:-36] == self.id: + return True + return False + + def get_crawled_items(self, root_id): + crawled_items = self.get_crawled_items_children(root_id) + crawled_items.append(root_id) + return crawled_items + + def get_crawled_items_children(self, root_id): + crawled_items = [] + for item_id in get_item_children(root_id): + if self.is_crawled_item(item_id): + crawled_items.append(item_id) + crawled_items.extend(self.get_crawled_items_children(self.id, item_id)) + return crawled_items + + def get_all_urls(self, date=False): ## parameters to add first_seen/last_seen ?????????????????????????????? + if date: + urls = {} + else: + urls = set() + for port in self.get_ports(): + for history in self.get_history_by_port(port, root=True): + if history.get('root'): + for item_id in self.get_crawled_items(history.get('root')): + url = get_item_url(item_id) + if url: + if date: + item_date = int(get_item_date(item_id)) + if url not in urls: + urls[url] = {'first_seen': item_date,'last_seen': item_date} + else: # update first_seen / last_seen + if item_date < urls[url]['first_seen']: + all_url[url]['first_seen'] = item_date + if item_date > urls[url]['last_seen']: + all_url[url]['last_seen'] = item_date + else: + urls.add(url) + return urls + + def get_misp_object(self): + # create domain-ip obj + obj_attrs = [] + obj = MISPObject('domain-crawled', standalone=True) + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_check() + + obj_attrs.append( obj.add_attribute('domain', value=self.id) ) + urls = self.get_all_urls(date=True) + for url in urls: + attribute = obj.add_attribute('url', value=url) + attribute.first_seen = str(urls[url]['first_seen']) + attribute.last_seen = str(urls[url]['last_seen']) + obj_attrs.append( attribute ) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + ############################################################################ ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index f49f5320..d3011b31 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -2,6 +2,7 @@ # -*-coding:UTF-8 -* import base64 +import gzip import os import re import sys @@ -38,10 +39,9 @@ import Username from flask import url_for config_loader = ConfigLoader() -# get and sanityze PASTE DIRECTORY -# # TODO: rename PASTES_FOLDER -PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' -PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') +# # TODO: get and sanityze ITEMS DIRECTORY +ITEMS_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' +ITEMS_FOLDER = os.path.join(os.path.realpath(ITEMS_FOLDER), '') r_cache = config_loader.get_redis_conn("Redis_Cache") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") @@ -82,11 +82,11 @@ class Item(AbstractObject): def get_filename(self): # Creating the full filepath - filename = os.path.join(PASTES_FOLDER, self.id) + filename = os.path.join(ITEMS_FOLDER, self.id) filename = os.path.realpath(filename) # incorrect filename - if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER: + if not os.path.commonprefix([filename, ITEMS_FOLDER]) == ITEMS_FOLDER: return None else: return filename @@ -114,8 +114,53 @@ class Item(AbstractObject): payload = {'raw': self.get_gzip_content(b64=True)} return payload - # # TODO: - def create(self): + def set_origin(self): # set_parent ? + pass + + def add_duplicate(self): + pass + + def sanitize_id(self): + pass + + # # TODO: sanitize_id + # # TODO: check if already exists ? + # # TODO: check if duplicate + def save_on_disk(self, content, binary=True, compressed=False, base64=False): + if not binary: + content = content.encode() + if base64: + content = base64.standard_b64decode(content) + if not compressed: + content = gzip.compress(content) + + # # TODO: # FIXME: raise Exception id filename is None ###### + filename = self.get_filename() + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(filename, 'wb') as f: + f.write(content) + + + # # TODO: correlations + # + # content + # tags + # origin + # duplicate -> all item iterations ??? + # + def create(self, content, tags, origin=None, duplicate=None): + self.save_on_disk(content, binary=True, compressed=False, base64=False) + + # # TODO: + # for tag in tags: + # self.add_tag(tag) + + if origin: + + if duplicate: + pass # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ @@ -174,7 +219,7 @@ def get_basename(item_id): return os.path.basename(item_id) def get_item_id(full_path): - return full_path.replace(PASTES_FOLDER, '', 1) + return full_path.replace(ITEMS_FOLDER, '', 1) def get_item_filepath(item_id): return item_basic.get_item_filepath(item_id) @@ -192,7 +237,7 @@ def get_item_basename(item_id): return os.path.basename(item_id) def get_item_size(item_id): - return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2) + return round(os.path.getsize(os.path.join(ITEMS_FOLDER, item_id))/1024.0, 2) def get_item_encoding(item_id): return None @@ -561,11 +606,11 @@ def get_item_har(har_path): def get_item_filename(item_id): # Creating the full filepath - filename = os.path.join(PASTES_FOLDER, item_id) + filename = os.path.join(ITEMS_FOLDER, item_id) filename = os.path.realpath(filename) # incorrect filename - if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER: + if not os.path.commonprefix([filename, ITEMS_FOLDER]) == ITEMS_FOLDER: return None else: return filename @@ -712,7 +757,7 @@ def delete_domain_node(item_id): delete_item(child_id) -# if __name__ == '__main__': -# -# item = Item('') -# print(item.get_misp_object().to_json()) +if __name__ == '__main__': + content = 'test file content' + item = Item('tests/2020/01/02/test_save.gz') + item.save_on_disk(content, binary=False) diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index 9c80367b..8e0b0fd7 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -58,6 +58,24 @@ class Pgp(AbstractObject): icon = 'times' return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5} + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('pgp-meta') + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + + if self.subtype=='key': + obj_attrs.append( obj.add_attribute('key-id', value=self.id) ) + elif self.subtype=='name': + obj_attrs.append( obj.add_attribute('user-id-name', value=self.id) ) + else: # mail + obj_attrs.append( obj.add_attribute('user-id-email', value=self.id) ) + + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + ############################################################################ ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index 41aff1be..60fe22ea 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -3,30 +3,24 @@ import os import sys -import redis -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +from io import BytesIO -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_object import AbstractObject -from abstract_object import AbstractObject - -config_loader = ConfigLoader.ConfigLoader() +config_loader = ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") -HASH_DIR = config_loader.get_config_str('Directories', 'hash') +SCREENSHOT_FOLDER = config_loader.get_files_directory('screenshot') config_loader = None - -################################################################################ -################################################################################ -################################################################################ - class Screenshot(AbstractObject): """ AIL Screenshot Object. (strings) """ + # ID = SHA256 def __init__(self, id): super(Screenshot, self).__init__('screenshot', id) @@ -50,6 +44,33 @@ class Screenshot(AbstractObject): def get_svg_icon(self): return {'style': 'fas', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius':5} + def get_rel_path(self, add_extension=False): + rel_path = os.path.join(self.id[0:2], self.id[2:4], self.id[4:6], self.id[6:8], self.id[8:10], self.id[10:12], self.id[12:]) + if add_extension: + rel_path = f'{rel_path}.png' + return rel_path + + def get_filepath(self): + filename = os.path.join(SCREENSHOT_FOLDER, self.get_rel_path(add_extension=True)) + return os.path.realpath(filename) + + def get_file_content(self): + filepath = self.get_filepath() + with open(filepath, 'rb') as f: + file_content = BytesIO(f.read()) + return file_content + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('file') + + obj_attrs.append( obj.add_attribute('sha256', value=self.id) ) + obj_attrs.append( obj.add_attribute('attachment', value=self.id, data=self.get_file_content()) ) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + ############################################################################ ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index 3c6c32b7..958e3dce 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -5,13 +5,15 @@ import os import sys import redis +from flask import url_for +from pymisp import MISPObject + # sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader -from abstract_object import AbstractObject -from flask import url_for +from abstract_subtype_object import AbstractSubtypeObject config_loader = ConfigLoader.ConfigLoader() @@ -22,13 +24,13 @@ config_loader = None ################################################################################ ################################################################################ -class Username(AbstractObject): +class Username(AbstractSubtypeObject): """ AIL Username Object. (strings) """ def __init__(self, id, subtype): - super(Username, self).__init__('username', id, subtype=subtype) + super(Username, self).__init__('username', id, subtype) # def get_ail_2_ail_payload(self): # payload = {'raw': self.get_gzip_content(b64=True), @@ -59,6 +61,28 @@ class Username(AbstractObject): icon = '\uf007' return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} + def get_misp_object(self): + obj_attrs = [] + if self.subtype == 'telegram': + obj = MISPObject('telegram-account', standalone=True) + obj_attrs.append( obj.add_attribute('username', value=self.id) ) + + elif self.subtype == 'twitter': + obj = MISPObject('twitter-account', standalone=True) + obj_attrs.append( obj.add_attribute('name', value=self.id) ) + + else: + obj = MISPObject('user-account', standalone=True) + obj_attrs.append( obj.add_attribute('username', value=self.id) ) + + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + ############################################################################ ############################################################################ ############################################################################ @@ -71,4 +95,7 @@ class Username(AbstractObject): -#if __name__ == '__main__': +if __name__ == '__main__': + + obj = Username('ninechantw', 'telegram') + print(obj.get_misp_object().to_json()) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 660dff6a..8e14590c 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -17,8 +17,8 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from packages import Tag -from lib.Investigations import is_object_investigated, get_obj_investigations -from lib.Tracker import is_obj_tracked, get_obj_all_trackers +from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations +from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers # # TODO: ADD CORRELATION ENGINE @@ -90,6 +90,14 @@ class AbstractObject(ABC): else: investigations = get_obj_investigations(self.id, self.type, self.subtype) return investigations + + def delete_investigations(self): + if not self.subtype: + unregistred = delete_obj_investigations(self.id, self.type) + else: + unregistred = delete_obj_investigations(self.id, self.type, self.subtype) + return unregistred + #- Investigations -# ## Trackers ## @@ -100,12 +108,20 @@ class AbstractObject(ABC): def get_trackers(self): return get_obj_all_trackers(self.type, self.subtype, self.id) + def delete_trackers(self): + return delete_obj_trackers(self.type, self.subtype, self.id) + #- Investigations -# def _delete(self): # DELETE TAGS Tag.delete_obj_all_tags(self.id, self.type) - # # TODO: remove from investigations + # remove from tracker + self.delete_trackers() + # remove from investigations + self.delete_investigations() + + # # TODO: remove from correlation @abstractmethod def delete(self): @@ -121,6 +137,10 @@ class AbstractObject(ABC): # """ # pass + @abstractmethod + def get_link(self, flask_context=False): + pass + @abstractmethod def get_svg_icon(self): """ @@ -129,7 +149,7 @@ class AbstractObject(ABC): pass @abstractmethod - def get_link(self, flask_context=False): + def get_misp_object(self): pass # # TODO: diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py new file mode 100755 index 00000000..741df301 --- /dev/null +++ b/bin/lib/objects/abstract_subtype_object.py @@ -0,0 +1,155 @@ +# -*-coding:UTF-8 -* +""" +Base Class for AIL Objects +""" + +################################## +# Import External packages +################################## +import os +import sys +from abc import abstractmethod + +#from flask import url_for + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects.abstract_object import AbstractObject +from lib.ConfigLoader import ConfigLoader + +# LOAD CONFIG +config_loader = ConfigLoader() +r_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None + +# # TODO: ADD CORRELATION ENGINE + +# # FIXME: SAVE SUBTYPE NAMES ????? + +class AbstractSubtypeObject(AbstractObject): + """ + Abstract Subtype Object + """ + + def __init__(self, obj_type, id, subtype): + """ Abstract for all the AIL object + + :param obj_type: object type (item, ...) + :param id: Object ID + """ + self.id = id + self.type = obj_type + self.subtype = subtype + + # # TODO: # FIXME: REMOVE R_INT ???????????????????????????????????????????????????????????????????? + def get_first_seen(self, r_int=False): + first_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen') + if r_int: + if first_seen: + return int(first_seen) + else: + return 99999999 + else: + return first_seen + + # # TODO: # FIXME: REMOVE R_INT ???????????????????????????????????????????????????????????????????? + def get_last_seen(self, r_int=False): + last_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'last_seen') + if r_int: + if last_seen: + return int(last_seen) + else: + return 0 + else: + return last_seen + + def get_nb_seen(self): + return r_metadata.scard(f'set_{self.type}_{self.subtype}:{self.id}') + + # # TODO: CHECK RESULT + def get_nb_seen_by_date(self, date_day): + nb = r_metadata.hget(f'{self.type}:{self.subtype}:{date_day}', self.id) + if nb is None: + return 0 + else: + return int(nb) + + def _get_meta(self): + meta_dict = {} + meta_dict['first_seen'] = self.get_first_seen() + meta_dict['last_seen'] = self.get_last_seen() + meta_dict['nb_seen'] = self.get_nb_seen() + return meta_dict + + # def exists(self): + # res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id) + # if res is not None: + # return True + # else: + # return False + + def exists(self): + return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}') + + def set_first_seen(self, first_seen): + r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen', first_seen) + + def set_last_seen(self, last_seen): + r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'last_seen', last_seen) + + def update_daterange(self, date): + date = int(date) + # obj don't exit + if not self.exists(): + self.set_first_seen(date) + self.set_last_seen(date) + else: + first_seen = self.get_first_seen(r_int=True) + last_seen = self.get_last_seen(r_int=True) + if date < first_seen: + self.set_first_seen(date) + if date > last_seen: + self.set_last_seen(date) + + def add(self, date): + self.update_correlation_daterange() + # daily + r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) + # all type + r_metadata.zincrby(f'{self.type}_all:{self.subtype}', self.id, 1) + + + + + # # domain + # if item_basic.is_crawled(item_id): + # domain = item_basic.get_item_domain(item_id) + # self.save_domain_correlation(domain, subtype, obj_id) + + + + def _delete(self): + pass + + + #################################### + # + # _get_items + # get_metadata + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index d5346efb..c6312603 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -14,9 +14,9 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/objects')) +from CryptoCurrencies import CryptoCurrency from Decodeds import Decoded from Domains import Domain -from CryptoCurrencies import CryptoCurrency from Items import Item from Pgps import Pgp from Screenshots import Screenshot @@ -76,12 +76,69 @@ def get_objects_meta(l_dict_objs, icon=False, url=False, flask_context=False): l_meta.append(dict_meta) return l_meta +# # TODO: CHECK IF object already have an UUID +def get_misp_object(obj_type, subtype, id): + object = get_object(obj_type, subtype, id) + return object.get_misp_object() + +# get misp relationship +def get_objects_relationship(obj_1, obj2): + relationship = {} + obj_types = ( obj_1.get_type(), obj2.get_type() ) + + ############################################################## + # if ['cryptocurrency', 'pgp', 'username', 'decoded', 'screenshot']: + # {'relation': '', 'src':, 'dest':} + # relationship[relation] = + ############################################################## + if 'cryptocurrency' in obj_types: + relationship['relation'] = 'extracted-from' + if obj1_type == 'cryptocurrency': + relationship['src'] = obj1_id + relationship['dest'] = obj2_id + else: + relationship['src'] = obj2_id + relationship['dest'] = obj1_id + + elif 'pgp' in obj_types: + relationship['relation'] = 'extracted-from' + + elif 'username' in obj_types: + relationship['relation'] = 'extracted-from' + + elif 'decoded' in obj_types: + relationship['relation'] = 'included-in' + + elif 'screenshot' in obj_types: + relationship['relation'] = 'screenshot-of' + + elif 'domain' in obj_types: + relationship['relation'] = 'extracted-from' + + # default + else: + pass + + + + + + + return relationship + + + + +################################################################################ +################################################################################ +################################################################################ + +def delete_obj(obj_type, subtype, id): + object = get_object(obj_type, subtype, id) + return object.delete() ################################################################################ ################################################################################ ################################################################################ ################################################################################ ################################################################################ -################################################################################ -################################################################################ -################################################################################ diff --git a/bin/modules/Hosts.py b/bin/modules/Hosts.py index 5c89a5dc..9ecb2590 100755 --- a/bin/modules/Hosts.py +++ b/bin/modules/Hosts.py @@ -63,11 +63,13 @@ class Hosts(AbstractModule): content = item.get_content() hosts = regex_helper.regex_findall(self.module_name, self.redis_cache_key, self.host_regex, item.get_id(), content) - for host in hosts: - #print(host) + if hosts: + print(f'{len(hosts)} host {item.get_id()}') + for host in hosts: + #print(host) - msg = f'{host} {item.get_id()}' - self.send_message_to_queue(msg, 'Host') + msg = f'{host} {item.get_id()}' + self.send_message_to_queue(msg, 'Host') diff --git a/bin/packages/git_status.py b/bin/packages/git_status.py index fcd746f9..7b9c467c 100755 --- a/bin/packages/git_status.py +++ b/bin/packages/git_status.py @@ -9,6 +9,8 @@ TERMINAL_BLUE = '\33[94m' TERMINAL_BLINK = '\33[6m' TERMINAL_DEFAULT = '\033[0m' +REPO_ORIGIN = 'https://github.com/ail-project/ail-framework.git' + # set defaut_remote def set_default_remote(new_origin_url, verbose=False): process = subprocess.run(['git', 'remote', 'set-url', 'origin', new_origin_url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -144,6 +146,28 @@ def get_last_tag_from_remote(verbose=False): print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) return '' +def get_git_metadata(): + dict_git = {} + dict_git['current_branch'] = get_current_branch() + dict_git['is_clone'] = is_not_fork(REPO_ORIGIN) + dict_git['is_working_directory_clean'] = is_working_directory_clean() + dict_git['current_commit'] = get_last_commit_id_from_local() + dict_git['last_remote_commit'] = get_last_commit_id_from_remote() + dict_git['last_local_tag'] = get_last_tag_from_local() + dict_git['last_remote_tag'] = get_last_tag_from_remote() + + if dict_git['current_commit'] != dict_git['last_remote_commit']: + dict_git['new_git_update_available'] = True + else: + dict_git['new_git_update_available'] = False + + if dict_git['last_local_tag'] != dict_git['last_remote_tag']: + dict_git['new_git_version_available'] = True + else: + dict_git['new_git_version_available'] = False + + return dict_git + if __name__ == "__main__": get_last_commit_id_from_remote(verbose=True) get_last_commit_id_from_local(verbose=True) diff --git a/bin/update-background.py b/bin/update-background.py index 9acc086a..3a38fc82 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -11,58 +11,42 @@ Update AIL in the background import os import sys -import redis import subprocess -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_updates def launch_background_upgrade(version, l_script_name): - if r_serv.sismember('ail:to_update', version): - r_serv.delete('ail:update_error') - r_serv.set('ail:update_in_progress', version) - r_serv.set('ail:current_background_update', version) + if ail_updates.is_version_in_background_update(version): + ail_updates.start_background_update(version) for script_name in l_script_name: - r_serv.set('ail:current_background_script', script_name) - update_file = os.path.join(os.environ['AIL_HOME'], 'update', version, script_name) + ail_updates.set_current_background_update_script(script_name) + update_file = ail_updates.get_current_background_update_script_path(version, script_name) + + # # TODO: Get error output process = subprocess.run(['python' ,update_file]) - update_progress = r_serv.get('ail:current_background_script_stat') - #if update_progress: - # if int(update_progress) != 100: - # r_serv.set('ail:update_error', 'Update {} Failed'.format(version)) - update_progress = r_serv.get('ail:current_background_script_stat') - if update_progress: - if int(update_progress) == 100: - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') - r_serv.srem('ail:to_update', version) + update_progress = ail_updates.get_current_background_update_progress() + if update_progress == 100: + ail_updates.end_background_update_script() + # # TODO: Create Custom error + # 'Please relaunch the bin/update-background.py script' + # # TODO: Create Class background update -def clean_update_db(): - r_serv.delete('ail:update_error') - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') + ail_updates.end_background_update() if __name__ == "__main__": - config_loader = ConfigLoader.ConfigLoader() - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - if r_serv.scard('ail:to_update') == 0: - clean_update_db() - - - launch_background_upgrade('v1.5', ['Update-ARDB_Onions.py', 'Update-ARDB_Metadata.py', 'Update-ARDB_Tags.py', 'Update-ARDB_Tags_background.py', 'Update-ARDB_Onions_screenshots.py']) - launch_background_upgrade('v2.4', ['Update_domain.py']) - launch_background_upgrade('v2.6', ['Update_screenshots.py']) - launch_background_upgrade('v2.7', ['Update_domain_tags.py']) - launch_background_upgrade('v3.4', ['Update_domain.py']) - launch_background_upgrade('v3.7', ['Update_trackers.py']) + if not ail_updates.exits_background_update_to_launch(): + ail_updates.clear_background_update() + else: + launch_background_upgrade('v1.5', ['Update-ARDB_Onions.py', 'Update-ARDB_Metadata.py', 'Update-ARDB_Tags.py', 'Update-ARDB_Tags_background.py', 'Update-ARDB_Onions_screenshots.py']) + launch_background_upgrade('v2.4', ['Update_domain.py']) + launch_background_upgrade('v2.6', ['Update_screenshots.py']) + launch_background_upgrade('v2.7', ['Update_domain_tags.py']) + launch_background_upgrade('v3.4', ['Update_domain.py']) + launch_background_upgrade('v3.7', ['Update_trackers.py']) diff --git a/update/bin/ail_updater.py b/update/bin/ail_updater.py index 9be78e81..6edc8ea6 100755 --- a/update/bin/ail_updater.py +++ b/update/bin/ail_updater.py @@ -2,14 +2,15 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_updates +from lib.ConfigLoader import ConfigLoader class AIL_Updater(object): """docstring for AIL_Updater.""" @@ -18,15 +19,11 @@ class AIL_Updater(object): self.version = new_version self.start_time = time.time() - self.config = ConfigLoader.ConfigLoader() - self.r_serv = self.config.get_redis_conn("ARDB_DB") + self.config = ConfigLoader() + self.r_serv = self.config.get_redis_conn("Kvrocks_DB") self.f_version = float(self.version[1:]) - self.current_f_version = self.r_serv.get('ail:version') - if self.current_f_version: - self.current_f_version = float(self.current_f_version[1:]) - else: - self.current_f_version = 0 + self.current_f_version = ail_updates.get_ail_float_version() def update(self): """ @@ -38,11 +35,7 @@ class AIL_Updater(object): """ Update DB version """ - #Set current ail version - self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d")) - #Set current ail version - if self.f_version > self.current_f_version: - self.r_serv.set('ail:version', self.version) + ail_updates.add_ail_update(version) def run_update(self): self.update() diff --git a/update/bin/old_ail_updater.py b/update/bin/old_ail_updater.py new file mode 100755 index 00000000..9be78e81 --- /dev/null +++ b/update/bin/old_ail_updater.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +class AIL_Updater(object): + """docstring for AIL_Updater.""" + + def __init__(self, new_version): + self.version = new_version + self.start_time = time.time() + + self.config = ConfigLoader.ConfigLoader() + self.r_serv = self.config.get_redis_conn("ARDB_DB") + + self.f_version = float(self.version[1:]) + self.current_f_version = self.r_serv.get('ail:version') + if self.current_f_version: + self.current_f_version = float(self.current_f_version[1:]) + else: + self.current_f_version = 0 + + def update(self): + """ + AIL DB update + """ + pass + + def end_update(self): + """ + Update DB version + """ + #Set current ail version + self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d")) + #Set current ail version + if self.f_version > self.current_f_version: + self.r_serv.set('ail:version', self.version) + + def run_update(self): + self.update() + self.end_update() diff --git a/update/default_update/Update.py b/update/default_update/Update.py index c34f4cd1..dc074860 100755 --- a/update/default_update/Update.py +++ b/update/default_update/Update.py @@ -9,8 +9,11 @@ import argparse import datetime import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_updates if __name__ == '__main__': parser = argparse.ArgumentParser(description='AIL default update') @@ -23,15 +26,11 @@ if __name__ == '__main__': # remove space update_tag = args.tag.replace(' ', '') + if not ail_updates.check_version(update_tag): + parser.print_help() + print(f'Error: Invalid update tag {update_tag}) + sys.exit(0) start_deb = time.time() - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #Set current ail version - r_serv.set('ail:version', update_tag) - - #Set current ail version - r_serv.hset('ail:update_date', update_tag, datetime.datetime.now().strftime("%Y%m%d")) + ail_updates.add_ail_update(update_tag) diff --git a/update/v3.4/Update.py b/update/v3.4/Update.py index 67f74ca0..ab88cdbc 100755 --- a/update/v3.4/Update.py +++ b/update/v3.4/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v3.5/Update.py b/update/v3.5/Update.py index 5cb73eb5..7be17cbb 100755 --- a/update/v3.5/Update.py +++ b/update/v3.5/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v3.6/Update.py b/update/v3.6/Update.py index a2ac201b..505d90ef 100755 --- a/update/v3.6/Update.py +++ b/update/v3.6/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v3.7/Update.py b/update/v3.7/Update.py index a8baa092..ff75e790 100755 --- a/update/v3.7/Update.py +++ b/update/v3.7/Update.py @@ -13,7 +13,7 @@ import ConfigLoader import Tracker sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v4.0/Update.py b/update/v4.0/Update.py index bd6596b3..0c9fa88f 100755 --- a/update/v4.0/Update.py +++ b/update/v4.0/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v4.1/Update.py b/update/v4.1/Update.py index 8fad9c87..696fd719 100755 --- a/update/v4.1/Update.py +++ b/update/v4.1/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/update/v4.2/Update.py b/update/v4.2/Update.py index baecba0a..47c66170 100755 --- a/update/v4.2/Update.py +++ b/update/v4.2/Update.py @@ -12,7 +12,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +from old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 3ab8dddf..9450bab5 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -28,7 +28,11 @@ import Tag sys.path.append('./modules/') -from User import User +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.Users import User sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -47,6 +51,7 @@ from blueprints.objects_item import objects_item from blueprints.hunters import hunters from blueprints.old_endpoints import old_endpoints from blueprints.ail_2_ail_sync import ail_2_ail_sync +from blueprints.settings_b import settings_b Flask_dir = os.environ['AIL_FLASK'] @@ -107,6 +112,7 @@ app.register_blueprint(objects_item, url_prefix=baseUrl) app.register_blueprint(hunters, url_prefix=baseUrl) app.register_blueprint(old_endpoints, url_prefix=baseUrl) app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl) +app.register_blueprint(settings_b, url_prefix=baseUrl) # ========= =========# # ========= Cookie name ======== diff --git a/var/www/blueprints/ail_2_ail_sync.py b/var/www/blueprints/ail_2_ail_sync.py index 037f8e5f..ad0a60d0 100644 --- a/var/www/blueprints/ail_2_ail_sync.py +++ b/var/www/blueprints/ail_2_ail_sync.py @@ -17,7 +17,6 @@ sys.path.append('modules') import Flask_config # Import Role_Manager -from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import login_admin, login_analyst, login_read_only sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index b4d99b81..2f1fb29f 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -17,7 +17,6 @@ sys.path.append('modules') import Flask_config # Import Role_Manager -from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import login_admin, login_analyst, login_read_only sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) diff --git a/var/www/blueprints/root.py b/var/www/blueprints/root.py index 37858b94..36cf7bec 100644 --- a/var/www/blueprints/root.py +++ b/var/www/blueprints/root.py @@ -15,15 +15,16 @@ sys.path.append('modules') import Flask_config # Import Role_Manager -from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import login_admin, login_analyst -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -from User import User +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import Users +from lib.Users import User r_cache = Flask_config.r_cache -r_serv_db = Flask_config.r_serv_db -r_serv_tags = Flask_config.r_serv_tags # ============ BLUEPRINT ============ @@ -67,7 +68,7 @@ def login(): return render_template("login.html", error=error) if user and user.check_password(password): - if not check_user_role_integrity(user.get_id()): + if not Users.check_user_role_integrity(user.get_id()): error = 'Incorrect User ACL, Please contact your administrator' return render_template("login.html", error=error) login_user(user) ## TODO: use remember me ? @@ -113,9 +114,9 @@ def change_password(): if current_user.is_authenticated and password1!=None: if password1==password2: - if check_password_strength(password1): + if Users.check_password_strength(password1): user_id = current_user.get_id() - create_user_db(user_id , password1, update=True) + Users.create_user(user_id , password=password1, chg_passwd=False) # update Note # dashboard return redirect(url_for('dashboard.index', update_note=True)) diff --git a/var/www/blueprints/settings_b.py b/var/www/blueprints/settings_b.py new file mode 100644 index 00000000..d5440fd4 --- /dev/null +++ b/var/www/blueprints/settings_b.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: ail_investigations +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +# sys.path.append('modules') +# import Flask_config + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_updates +from packages import git_status + +# ============ BLUEPRINT ============ +settings_b = Blueprint('settings_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/settings')) + +# ============ VARIABLES ============ +#bootstrap_label = Flask_config.bootstrap_label + +# ============ FUNCTIONS ============ +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============= ROUTES ============== + +@settings_b.route("/settings", methods=['GET']) +@login_required +@login_read_only +def settings_page(): + git_metadata = git_status.get_git_metadata() + ail_version = ail_updates.get_ail_version() + #admin_level = current_user.is_in_role('admin') + return render_template("settings_index.html", git_metadata=git_metadata, + ail_version=ail_version) + +@settings_b.route("/settings/background_update/json", methods=['GET']) +@login_required +@login_read_only +def get_background_update_metadata_json(): + return jsonify(ail_updates.get_update_background_metadata()) + + + + + + + + + + + +############################################# diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py index 3aab97aa..deadd829 100755 --- a/var/www/create_default_user.py +++ b/var/www/create_default_user.py @@ -5,42 +5,29 @@ import os import sys import redis -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) - -from Role_Manager import create_user_db, edit_user_db, get_default_admin_token, gen_password - -config_loader = ConfigLoader.ConfigLoader() - -r_serv = config_loader.get_redis_conn("ARDB_DB") -config_loader = None +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import Users if __name__ == "__main__": # create role_list - if not r_serv.exists('ail:all_role'): - r_serv.zadd('ail:all_role', 1, 'admin') - r_serv.zadd('ail:all_role', 2, 'analyst') - r_serv.zadd('ail:all_role', 3, 'user') - r_serv.zadd('ail:all_role', 4, 'user_no_api') - r_serv.zadd('ail:all_role', 5, 'read_only') + Users._create_roles_list() - username = 'admin@admin.test' - password = gen_password() - if r_serv.exists('user_metadata:admin@admin.test'): - edit_user_db(username, password=password, role='admin') - else: - create_user_db(username, password, role='admin', default=True) - token = get_default_admin_token() + user_id = 'admin@admin.test' + password = Users.gen_password() + + create_user(user_id, password=password, role='admin') + token = Users.get_default_admin_token() default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') to_write_str = '# Password Generated by default\n# This file is deleted after the first login\n#\nemail=admin@admin.test\npassword=' - to_write_str = to_write_str + password + '\nAPI_Key=' + token + '\n' + to_write_str = f'{to_write_str}{password}\nAPI_Key={token}\n' with open(default_passwd_file, 'w') as f: f.write(to_write_str) - print('new user created: {}'.format(username)) + print('new user created: {}'.format(user_id)) print('password: {}'.format(password)) print('token: {}'.format(token)) diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py index 080a8ba6..003a89cc 100644 --- a/var/www/modules/Role_Manager.py +++ b/var/www/modules/Role_Manager.py @@ -7,9 +7,6 @@ import sys import redis import bcrypt -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - from functools import wraps from flask_login import LoginManager, current_user, login_user, logout_user, login_required @@ -18,17 +15,6 @@ from flask import request, make_response, current_app login_manager = LoginManager() login_manager.login_view = 'root.role' -# CONFIG # -config_loader = ConfigLoader.ConfigLoader() - -r_serv_db = config_loader.get_redis_conn("ARDB_DB") -config_loader = None - -default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') - -regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,100}$' -regex_password = re.compile(regex_password) - ############################################################### ############### FLASK CACHE ################## ############################################################### @@ -100,138 +86,3 @@ def login_read_only(func): ############################################################### ############################################################### -############################################################### - -def gen_password(length=30, charset="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_!@#$%^&*()"): - random_bytes = os.urandom(length) - len_charset = len(charset) - indices = [int(len_charset * (byte / 256.0)) for byte in random_bytes] - return "".join([charset[index] for index in indices]) - -def gen_token(length=41, charset="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"): - random_bytes = os.urandom(length) - len_charset = len(charset) - indices = [int(len_charset * (byte / 256.0)) for byte in random_bytes] - return "".join([charset[index] for index in indices]) - -def generate_new_token(user_id): - # create user token - current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token') - r_serv_db.hdel('user:tokens', current_token) - token = gen_token(41) - r_serv_db.hset('user:tokens', token, user_id) - r_serv_db.hset('user_metadata:{}'.format(user_id), 'token', token) - -def get_default_admin_token(): - if r_serv_db.exists('user_metadata:admin@admin.test'): - return r_serv_db.hget('user_metadata:admin@admin.test', 'token') - else: - return '' - -def create_user_db(username_id , password, default=False, role=None, update=False): - password = password.encode() - password_hash = hashing_password(password) - - # create user token - generate_new_token(username_id) - - if not role: - role = 'read_only' - - if update: - r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd') - # remove default user password file - if username_id=='admin@admin.test': - os.remove(default_passwd_file) - r_serv_db.hset('user:all', username_id, password_hash) - else: - if default: - r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True) - if role: - if role in get_all_role(): - for role_to_add in get_all_user_role(role): - r_serv_db.sadd('user_role:{}'.format(role_to_add), username_id) - r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role) - - r_serv_db.hset('user:all', username_id, password_hash) - -def edit_user_db(user_id, role, password=None): - if password: - password_hash = hashing_password(password.encode()) - r_serv_db.hset('user:all', user_id, password_hash) - - current_role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role') - if role != current_role: - request_level = get_role_level(role) - current_role = get_role_level(current_role) - - if current_role < request_level: - role_to_remove = get_user_role_by_range(current_role -1, request_level - 2) - for role_id in role_to_remove: - r_serv_db.srem('user_role:{}'.format(role_id), user_id) - r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role) - else: - role_to_add = get_user_role_by_range(request_level -1, current_role) - for role_id in role_to_add: - r_serv_db.sadd('user_role:{}'.format(role_id), user_id) - r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role) - -def delete_user_db(user_id): - if r_serv_db.exists('user_metadata:{}'.format(user_id)): - role_to_remove =get_all_role() - for role_id in role_to_remove: - r_serv_db.srem('user_role:{}'.format(role_id), user_id) - user_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token') - r_serv_db.hdel('user:tokens', user_token) - r_serv_db.delete('user_metadata:{}'.format(user_id)) - r_serv_db.hdel('user:all', user_id) - else: - print('Error: user {} do not exist'.format(user_id)) - -def hashing_password(bytes_password): - hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt()) - return hashed - -def check_password_strength(password): - result = regex_password.match(password) - if result: - return True - else: - return False - -def get_all_role(): - return r_serv_db.zrange('ail:all_role', 0, -1) - -def get_role_level(role): - return int(r_serv_db.zscore('ail:all_role', role)) - -def get_all_user_role(user_role): - current_role_val = get_role_level(user_role) - return r_serv_db.zrange('ail:all_role', current_role_val -1, -1) - -def get_all_user_upper_role(user_role): - current_role_val = get_role_level(user_role) - # remove one rank - if current_role_val > 1: - return r_serv_db.zrange('ail:all_role', 0, current_role_val -2) - else: - return [] - -def get_user_role_by_range(inf, sup): - return r_serv_db.zrange('ail:all_role', inf, sup) - -def get_user_role(user_id): - return r_serv_db.hget('user_metadata:{}'.format(user_id), 'role') - -def check_user_role_integrity(user_id): - user_role = get_user_role(user_id) - all_user_role = get_all_user_role(user_role) - res = True - for role in all_user_role: - if not r_serv_db.sismember('user_role:{}'.format(role), user_id): - res = False - upper_role = get_all_user_upper_role(user_role) - for role in upper_role: - if r_serv_db.sismember('user_role:{}'.format(role), user_id): - res = False - return res diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index d690cd59..92a28772 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -20,6 +20,7 @@ from flask_login import login_required sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import queues_modules +import ail_updates # ============ VARIABLES ============ import Flask_config @@ -29,10 +30,8 @@ config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv = Flask_config.r_serv r_serv_log = Flask_config.r_serv_log -r_serv_db = Flask_config.r_serv_db max_dashboard_logs = Flask_config.max_dashboard_logs -dict_update_description = Flask_config.dict_update_description dashboard = Blueprint('dashboard', __name__, template_folder='templates') @@ -151,12 +150,11 @@ def stuff(): return jsonify(row1=get_queues(r_serv)) +# TODO: ADD UPDATE NOTE BY USER @dashboard.route("/") @login_required @login_read_only def index(): - update_note = request.args.get('update_note') - default_minute = config_loader.get_config_str("Flask", "minute_processed_paste") threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module") log_select = {10, 25, 50, 100} @@ -165,21 +163,15 @@ def index(): log_select.sort() # Check if update in progress - update_in_progress = False - update_warning_message = '' - update_warning_message_notice_me = '' - current_update = r_serv_db.get('ail:current_background_update') - if current_update: - if r_serv_db.scard('ail:update_{}'.format(current_update)) != dict_update_description[current_update]['nb_background_update']: - update_in_progress = True - update_warning_message = dict_update_description[current_update]['update_warning_message'] - update_warning_message_notice_me = dict_update_description[current_update]['update_warning_message_notice_me'] + background_update = False + update_message = '' + if ail_updates.get_current_background_update(): + background_update = True + update_message = ail_updates.get_update_background_message() return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module, log_select=log_select, selected=max_dashboard_logs, - update_warning_message=update_warning_message, update_in_progress=update_in_progress, - #update_note=update_note, - update_warning_message_notice_me=update_warning_message_notice_me) + background_update=background_update, update_message=update_message) # ========= REGISTRATION ========= app.register_blueprint(dashboard, url_prefix=baseUrl) diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index 35d5188f..3e86f4fa 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -62,10 +62,11 @@
- {%if update_in_progress%} + {%if background_update%}