mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-09-19 23:48:23 +00:00
fix: [Tracker migration] clean old Term library
This commit is contained in:
Terrtia
parent
41a4278371
commit
8672671e51
5 changed files with 92 additions and 57 deletions
|
|
@ -220,8 +220,11 @@ def tags_migration():
|
|||
|
||||
pass
|
||||
|
||||
def items_migration():
|
||||
pass
|
||||
|
||||
def crawler_migration():
|
||||
|
||||
|
||||
pass
|
||||
|
||||
def domain_migration():
|
||||
|
|
@ -244,7 +247,7 @@ if __name__ == '__main__':
|
|||
core_migration()
|
||||
user_migration()
|
||||
#ail_2_ail_migration()
|
||||
#trackers_migration()
|
||||
trackers_migration()
|
||||
#investigations_migration()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -92,6 +92,18 @@ def get_all_tracker_by_type(tracker_type):
|
|||
def get_all_tracker_by_type(tracker_type):
|
||||
return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
|
||||
|
||||
def get_user_trackers_uuid(user_id, tracker_type=None):
|
||||
if tracker_type:
|
||||
return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}')
|
||||
else:
|
||||
return r_serv_tracker.smembers(f'user:tracker:{user_id}')
|
||||
|
||||
def get_global_trackers_uuid(tracker_type=None):
|
||||
if tracker_type:
|
||||
return r_serv_tracker.smembers(f'global:tracker:{tracker_type}')
|
||||
else:
|
||||
return r_serv_tracker.smembers('global:tracker')
|
||||
|
||||
def get_tracker_by_uuid(tracker_uuid):
|
||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
||||
|
||||
|
|
@ -169,6 +181,18 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F
|
|||
|
||||
return dict_uuid
|
||||
|
||||
def get_user_trackers_metadata(user_id, tracker_type=None):
|
||||
meta_trackers = []
|
||||
for tracker_uuid in get_user_trackers_uuid(user_id, tracker_type=None):
|
||||
meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True))
|
||||
return meta_trackers
|
||||
|
||||
def get_global_trackers_metadata(tracker_type=None):
|
||||
meta_trackers = []
|
||||
for tracker_uuid in get_global_trackers_uuid(tracker_type=None):
|
||||
meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True))
|
||||
return meta_trackers
|
||||
|
||||
def get_tracker_metadata_api(request_dict):
|
||||
tracker_uuid = request_dict.get('tracker_uuid', None)
|
||||
if not request_dict:
|
||||
|
|
@ -326,6 +350,14 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
|
|||
return True
|
||||
return False
|
||||
|
||||
## API ##
|
||||
def api_check_tracker_uuid(tracker_uuid):
|
||||
if not is_valid_uuid_v4(task_uuid):
|
||||
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
return None
|
||||
|
||||
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
||||
if not is_valid_uuid_v4(tracker_uuid):
|
||||
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||
|
|
@ -1135,7 +1167,7 @@ def save_retro_hunt_match(task_uuid, id, object_type='item'):
|
|||
if res == 1:
|
||||
r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1)
|
||||
# Add map obj_id -> task_uuid
|
||||
r_serv_tracker.sadd(f'obj:retro_hunt:item:{item_id}', task_uuid)
|
||||
r_serv_tracker.sadd(f'obj:retro_hunt:item:{id}', task_uuid)
|
||||
|
||||
def get_retro_hunt_all_item_dates(task_uuid):
|
||||
return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import Date
|
|||
import Item
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
|
||||
r_serv_term = config_loader.get_redis_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
||||
|
|
|
|||
|
|
@ -44,49 +44,49 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
|
|||
@login_read_only
|
||||
def tracked_menu():
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id)
|
||||
global_term = Term.get_all_global_tracked_terms()
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label)
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id)
|
||||
global_trackers = Tracker.get_global_trackers_metadata()
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunter.route("/trackers/word")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_word():
|
||||
filter_type = 'word'
|
||||
tracker_type = 'word'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type='word')
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type='word')
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/set")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_set():
|
||||
filter_type = 'set'
|
||||
tracker_type = 'set'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/regex")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_regex():
|
||||
filter_type = 'regex'
|
||||
tracker_type = 'regex'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/yara")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_yara():
|
||||
filter_type = 'yara'
|
||||
tracker_type = 'yara'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
|
||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||
|
|
@ -119,7 +119,7 @@ def add_tracked_menu():
|
|||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except Exception:
|
||||
galaxies_tags = []
|
||||
|
||||
|
||||
|
||||
# YARA #
|
||||
if tracker_type == 'yara':
|
||||
|
|
@ -170,8 +170,8 @@ def edit_tracked_menu():
|
|||
user_id = current_user.get_id()
|
||||
tracker_uuid = request.args.get('uuid', None)
|
||||
|
||||
res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) # check if is author or admin
|
||||
if res: # invalid access
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id) # check if is author or admin
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True)
|
||||
|
|
@ -209,8 +209,8 @@ def edit_tracked_menu():
|
|||
def show_tracker():
|
||||
user_id = current_user.get_id()
|
||||
tracker_uuid = request.args.get('uuid', None)
|
||||
res = Term.check_term_uuid_valid_access(tracker_uuid, user_id)
|
||||
if res: # invalid access
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
date_from = request.args.get('date_from')
|
||||
|
|
@ -252,8 +252,8 @@ def show_tracker():
|
|||
def update_tracker_description():
|
||||
user_id = current_user.get_id()
|
||||
term_uuid = request.form.get('uuid')
|
||||
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||
if res: # invalid access
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
description = escape( str(request.form.get('description', '')) )
|
||||
Term.replace_tracker_description(term_uuid, description)
|
||||
|
|
@ -265,8 +265,8 @@ def update_tracker_description():
|
|||
def update_tracker_tags():
|
||||
user_id = current_user.get_id()
|
||||
term_uuid = request.form.get('uuid')
|
||||
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||
if res: # invalid access
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
tags = request.form.get('tags')
|
||||
if tags:
|
||||
|
|
@ -282,8 +282,8 @@ def update_tracker_tags():
|
|||
def update_tracker_mails():
|
||||
user_id = current_user.get_id()
|
||||
term_uuid = request.form.get('uuid')
|
||||
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||
if res: # invalid access
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
mails = request.form.get('mails')
|
||||
if mails:
|
||||
|
|
|
|||
|
|
@ -46,9 +46,11 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% if user_trackers %}
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header">
|
||||
<h5 class="card-title">Your {{ filter_type }} Trackers</h5>
|
||||
<h5 class="card-title">Your {{ tracker_type }} Trackers</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<table id="table_user_trackers" class="table table-striped table-bordered">
|
||||
|
|
@ -63,17 +65,17 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for dict_uuid in user_term %}
|
||||
{% for dict_uuid in user_trackers %}
|
||||
<tr>
|
||||
<td>{{ dict_uuid['type'] }}</td>
|
||||
<td>
|
||||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['term'] %}
|
||||
{% if dict_uuid['term']|length > 256 %}
|
||||
{{ dict_uuid['term'][0:256] }}...
|
||||
{% if dict_uuid['tracker'] %}
|
||||
{% if dict_uuid['tracker']|length > 256 %}
|
||||
{{ dict_uuid['tracker'][0:256] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{{ dict_uuid['tracker'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
@ -88,14 +90,12 @@
|
|||
</td>
|
||||
<td>
|
||||
{% if dict_uuid['first_seen'] %}
|
||||
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/
|
||||
{{ dict_uuid['first_seen'][6:8] }}
|
||||
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if dict_uuid['last_seen'] %}
|
||||
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/
|
||||
{{ dict_uuid['last_seen'][6:8] }}
|
||||
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
|
|
@ -111,9 +111,11 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% endif %}
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header">
|
||||
<h5 class="card-title">Global {{ filter_type }} Trackers</h5>
|
||||
<h5 class="card-title">Global {{ tracker_type }} Trackers</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<table id="table_global_trackers" class="table table-striped table-bordered">
|
||||
|
|
@ -128,17 +130,17 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for dict_uuid in global_term %}
|
||||
{% for dict_uuid in global_trackers %}
|
||||
<tr>
|
||||
<td>{{ dict_uuid['type'] }}</td>
|
||||
<td>
|
||||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['term'] %}
|
||||
{% if dict_uuid['term']|length > 256 %}
|
||||
{{ dict_uuid['term'][0:256] }}...
|
||||
{% if dict_uuid['tracker'] %}
|
||||
{% if dict_uuid['tracker']|length > 256 %}
|
||||
{{ dict_uuid['tracker'][0:256] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{{ dict_uuid['tracker'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
@ -153,14 +155,12 @@
|
|||
</td>
|
||||
<td>
|
||||
{% if dict_uuid['first_seen'] %}
|
||||
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/
|
||||
{{ dict_uuid['first_seen'][6:8] }}
|
||||
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if dict_uuid['last_seen'] %}
|
||||
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/
|
||||
{{ dict_uuid['last_seen'][6:8] }}
|
||||
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
|
||||
|
|
@ -190,12 +190,12 @@
|
|||
<script>
|
||||
$(document).ready(function () {
|
||||
$("#page-Tracker").addClass("active");
|
||||
$("#nav_tracker_{{filter_type}}").addClass("active");
|
||||
$("#nav_tracker_{{tracker_type}}").addClass("active");
|
||||
|
||||
{% for dict_uuid in user_term %}
|
||||
{% for dict_uuid in user_trackers %}
|
||||
sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});
|
||||
{% endfor %}
|
||||
{% for dict_uuid in global_term %}
|
||||
{% for dict_uuid in global_trackers %}
|
||||
sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});
|
||||
{% endfor %}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue