diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index e5f678df..6f6cc3e9 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -220,8 +220,11 @@ def tags_migration(): pass +def items_migration(): + pass + def crawler_migration(): - + pass def domain_migration(): @@ -244,7 +247,7 @@ if __name__ == '__main__': core_migration() user_migration() #ail_2_ail_migration() - #trackers_migration() + trackers_migration() #investigations_migration() diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 589581c3..e26f3861 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -92,6 +92,18 @@ def get_all_tracker_by_type(tracker_type): def get_all_tracker_by_type(tracker_type): return r_serv_tracker.smembers(f'all:tracker:{tracker_type}') +def get_user_trackers_uuid(user_id, tracker_type=None): + if tracker_type: + return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') + else: + return r_serv_tracker.smembers(f'user:tracker:{user_id}') + +def get_global_trackers_uuid(tracker_type=None): + if tracker_type: + return r_serv_tracker.smembers(f'global:tracker:{tracker_type}') + else: + return r_serv_tracker.smembers('global:tracker') + def get_tracker_by_uuid(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked') @@ -169,6 +181,18 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F return dict_uuid +def get_user_trackers_metadata(user_id, tracker_type=None): + meta_trackers = [] + for tracker_uuid in get_user_trackers_uuid(user_id, tracker_type=None): + meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True)) + return meta_trackers + +def get_global_trackers_metadata(tracker_type=None): + meta_trackers = [] + for tracker_uuid in get_global_trackers_uuid(tracker_type=None): + meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True)) + return meta_trackers + def get_tracker_metadata_api(request_dict): tracker_uuid = request_dict.get('tracker_uuid', None) if not request_dict: @@ -326,6 +350,14 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id): return True return False +## API ## +def api_check_tracker_uuid(tracker_uuid): + if not is_valid_uuid_v4(task_uuid): + return {"status": "error", "reason": "Invalid uuid"}, 400 + if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): + return {"status": "error", "reason": "Unknown uuid"}, 404 + return None + def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 @@ -1135,7 +1167,7 @@ def save_retro_hunt_match(task_uuid, id, object_type='item'): if res == 1: r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1) # Add map obj_id -> task_uuid - r_serv_tracker.sadd(f'obj:retro_hunt:item:{item_id}', task_uuid) + r_serv_tracker.sadd(f'obj:retro_hunt:item:{id}', task_uuid) def get_retro_hunt_all_item_dates(task_uuid): return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index f46ec4ea..6e9b9b8e 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -25,7 +25,7 @@ import Date import Item config_loader = ConfigLoader.ConfigLoader() -r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") +r_serv_term = config_loader.get_redis_conn("Kvrocks_DB") config_loader = None email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 12d40cfe..208b1708 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -44,49 +44,49 @@ hunter = Blueprint('hunter', __name__, template_folder='templates') @login_read_only def tracked_menu(): user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id) - global_term = Term.get_all_global_tracked_terms() - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label) + user_trackers = Tracker.get_user_trackers_metadata(user_id) + global_trackers = Tracker.get_global_trackers_metadata() + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label) @hunter.route("/trackers/word") @login_required @login_read_only def tracked_menu_word(): - filter_type = 'word' + tracker_type = 'word' user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id, filter_type='word') - global_term = Term.get_all_global_tracked_terms(filter_type='word') - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word') + global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word') + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) @hunter.route("/trackers/set") @login_required @login_read_only def tracked_menu_set(): - filter_type = 'set' + tracker_type = 'set' user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) - global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) @hunter.route("/trackers/regex") @login_required @login_read_only def tracked_menu_regex(): - filter_type = 'regex' + tracker_type = 'regex' user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) - global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) @hunter.route("/trackers/yara") @login_required @login_read_only def tracked_menu_yara(): - filter_type = 'yara' + tracker_type = 'yara' user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) - global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type) @hunter.route("/tracker/add", methods=['GET', 'POST']) @@ -119,7 +119,7 @@ def add_tracked_menu(): galaxies_tags = json.loads(galaxies_tags) except Exception: galaxies_tags = [] - + # YARA # if tracker_type == 'yara': @@ -170,8 +170,8 @@ def edit_tracked_menu(): user_id = current_user.get_id() tracker_uuid = request.args.get('uuid', None) - res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) # check if is author or admin - if res: # invalid access + res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id) # check if is author or admin + if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) @@ -209,8 +209,8 @@ def edit_tracked_menu(): def show_tracker(): user_id = current_user.get_id() tracker_uuid = request.args.get('uuid', None) - res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) - if res: # invalid access + res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id) + if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] date_from = request.args.get('date_from') @@ -252,8 +252,8 @@ def show_tracker(): def update_tracker_description(): user_id = current_user.get_id() term_uuid = request.form.get('uuid') - res = Term.check_term_uuid_valid_access(term_uuid, user_id) - if res: # invalid access + res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id) + if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] description = escape( str(request.form.get('description', '')) ) Term.replace_tracker_description(term_uuid, description) @@ -265,8 +265,8 @@ def update_tracker_description(): def update_tracker_tags(): user_id = current_user.get_id() term_uuid = request.form.get('uuid') - res = Term.check_term_uuid_valid_access(term_uuid, user_id) - if res: # invalid access + res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id) + if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] tags = request.form.get('tags') if tags: @@ -282,8 +282,8 @@ def update_tracker_tags(): def update_tracker_mails(): user_id = current_user.get_id() term_uuid = request.form.get('uuid') - res = Term.check_term_uuid_valid_access(term_uuid, user_id) - if res: # invalid access + res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id) + if res[1] != 200: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] mails = request.form.get('mails') if mails: diff --git a/var/www/modules/hunter/templates/trackersManagement.html b/var/www/modules/hunter/templates/trackersManagement.html index aedc790d..52ced6af 100644 --- a/var/www/modules/hunter/templates/trackersManagement.html +++ b/var/www/modules/hunter/templates/trackersManagement.html @@ -46,9 +46,11 @@
+ {% if user_trackers %} +
-
Your {{ filter_type }} Trackers
+
Your {{ tracker_type }} Trackers
@@ -63,17 +65,17 @@ - {% for dict_uuid in user_term %} + {% for dict_uuid in user_trackers %}
{{ dict_uuid['type'] }} - {% if dict_uuid['term'] %} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256] }}... + {% if dict_uuid['tracker'] %} + {% if dict_uuid['tracker']|length > 256 %} + {{ dict_uuid['tracker'][0:256] }}... {% else %} - {{ dict_uuid['term'] }} + {{ dict_uuid['tracker'] }} {% endif %} {% endif %} @@ -88,14 +90,12 @@ {% if dict_uuid['first_seen'] %} - {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ - {{ dict_uuid['first_seen'][6:8] }} + {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }} {% endif %} {% if dict_uuid['last_seen'] %} - {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ - {{ dict_uuid['last_seen'][6:8] }} + {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }} {% endif %} @@ -111,9 +111,11 @@ + {% endif %} +
-
Global {{ filter_type }} Trackers
+
Global {{ tracker_type }} Trackers
@@ -128,17 +130,17 @@ - {% for dict_uuid in global_term %} + {% for dict_uuid in global_trackers %} @@ -190,12 +190,12 @@
{{ dict_uuid['type'] }} - {% if dict_uuid['term'] %} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256] }}... + {% if dict_uuid['tracker'] %} + {% if dict_uuid['tracker']|length > 256 %} + {{ dict_uuid['tracker'][0:256] }}... {% else %} - {{ dict_uuid['term'] }} + {{ dict_uuid['tracker'] }} {% endif %} {% endif %} @@ -153,14 +155,12 @@ {% if dict_uuid['first_seen'] %} - {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ - {{ dict_uuid['first_seen'][6:8] }} + {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }} {% endif %} {% if dict_uuid['last_seen'] %} - {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ - {{ dict_uuid['last_seen'][6:8] }} + {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }} {% endif %}