mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-09-20 07:58:29 +00:00
chg: [UI term tracker] refractor term management: trackers list + show trackers + add new trackers
This commit is contained in:
Terrtia
parent
80f9535074
commit
7ed09bc923
8 changed files with 865 additions and 356 deletions
|
|
@ -17,6 +17,8 @@ from textblob import TextBlob
|
||||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||||
import Flask_config
|
import Flask_config
|
||||||
|
|
||||||
|
import Date
|
||||||
|
|
||||||
r_serv_term = Flask_config.r_serv_term
|
r_serv_term = Flask_config.r_serv_term
|
||||||
email_regex = Flask_config.email_regex
|
email_regex = Flask_config.email_regex
|
||||||
|
|
||||||
|
|
@ -235,7 +237,7 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, dashboard=0)
|
||||||
if level == 0: # user only
|
if level == 0: # user only
|
||||||
r_serv_term.sadd('user:tracked_term:{}'.format(user_id), term_uuid)
|
r_serv_term.sadd('user:tracked_term:{}'.format(user_id), term_uuid)
|
||||||
elif level == 1: # global
|
elif level == 1: # global
|
||||||
r_serv_term.sadd('gobal:tracked_term', term_uuid)
|
r_serv_term.sadd('global:tracked_term', term_uuid)
|
||||||
|
|
||||||
# create term tags list
|
# create term tags list
|
||||||
for tag in tags:
|
for tag in tags:
|
||||||
|
|
@ -274,7 +276,7 @@ def delete_term(term_uuid):
|
||||||
user_id = term_type = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id')
|
user_id = term_type = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id')
|
||||||
r_serv_term.srem('user:tracked_term:{}'.format(user_id), term_uuid)
|
r_serv_term.srem('user:tracked_term:{}'.format(user_id), term_uuid)
|
||||||
elif level == 1: # global
|
elif level == 1: # global
|
||||||
r_serv_term.srem('gobal:tracked_term', term_uuid)
|
r_serv_term.srem('global:tracked_term', term_uuid)
|
||||||
|
|
||||||
# delete metatadata
|
# delete metatadata
|
||||||
r_serv_term.delete('tracked_term:{}'.format(term_uuid))
|
r_serv_term.delete('tracked_term:{}'.format(term_uuid))
|
||||||
|
|
@ -291,6 +293,20 @@ def delete_term(term_uuid):
|
||||||
r_serv_term.delete('tracked_term:item:{}:{}'.format(term_uuid, date))
|
r_serv_term.delete('tracked_term:item:{}:{}'.format(term_uuid, date))
|
||||||
r_serv_term.delete('tracked_term:stat:{}'.format(term_uuid))
|
r_serv_term.delete('tracked_term:stat:{}'.format(term_uuid))
|
||||||
|
|
||||||
|
def replace_tracked_term_tags(term_uuid, tags):
|
||||||
|
r_serv_term.delete('tracked_term:tags:{}'.format(term_uuid))
|
||||||
|
for tag in tags:
|
||||||
|
r_serv_term.sadd('tracked_term:tags:{}'.format(term_uuid), tag)
|
||||||
|
|
||||||
|
def replace_tracked_term_mails(term_uuid, mails):
|
||||||
|
res = verify_mail_list(mails)
|
||||||
|
if res:
|
||||||
|
return res
|
||||||
|
else:
|
||||||
|
r_serv_term.delete('tracked_term:mail:{}'.format(term_uuid))
|
||||||
|
for mail in mails:
|
||||||
|
r_serv_term.sadd('tracked_term:mail:{}'.format(term_uuid), mail)
|
||||||
|
|
||||||
def get_term_uuid_list(term, term_type):
|
def get_term_uuid_list(term, term_type):
|
||||||
return list(r_serv_term.smembers('all:tracked_term_uuid:{}:{}'.format(term_type, term)))
|
return list(r_serv_term.smembers('all:tracked_term_uuid:{}:{}'.format(term_type, term)))
|
||||||
|
|
||||||
|
|
@ -336,7 +352,7 @@ def parse_get_tracker_term_item(dict_input, user_id):
|
||||||
date_to = dict_input.get('date_to', None)
|
date_to = dict_input.get('date_to', None)
|
||||||
|
|
||||||
if date_from is None:
|
if date_from is None:
|
||||||
date_from = r_serv_term.zrevrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
|
date_from = get_tracked_term_first_seen(term_uuid)
|
||||||
if date_from:
|
if date_from:
|
||||||
date_from = date_from[0]
|
date_from = date_from[0]
|
||||||
|
|
||||||
|
|
@ -355,45 +371,83 @@ def parse_get_tracker_term_item(dict_input, user_id):
|
||||||
res_dict['items'] = all_item_id
|
res_dict['items'] = all_item_id
|
||||||
return (res_dict, 200)
|
return (res_dict, 200)
|
||||||
|
|
||||||
|
def get_tracked_term_first_seen(term_uuid):
|
||||||
|
res = r_serv_term.zrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
|
||||||
|
if res:
|
||||||
|
return res[0]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_tracked_term_last_seen(term_uuid):
|
||||||
|
res = r_serv_term.zrevrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
|
||||||
|
if res:
|
||||||
|
return res[0]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_term_metedata(term_uuid, user_id=False, level=False, tags=False, mails=False, sparkline=False):
|
||||||
|
dict_uuid = {}
|
||||||
|
dict_uuid['term'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'tracked')
|
||||||
|
dict_uuid['type'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'type')
|
||||||
|
dict_uuid['date'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'date')
|
||||||
|
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
|
||||||
|
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
|
||||||
|
if user_id:
|
||||||
|
dict_uuid['user_id'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id')
|
||||||
|
if level:
|
||||||
|
dict_uuid['level'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'level')
|
||||||
|
if mails:
|
||||||
|
dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
|
||||||
|
if tags:
|
||||||
|
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
|
||||||
|
if sparkline:
|
||||||
|
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
|
||||||
|
dict_uuid['uuid'] = term_uuid
|
||||||
|
return dict_uuid
|
||||||
|
|
||||||
|
def get_tracked_term_sparkline(term_uuid, num_day=6):
|
||||||
|
date_range_sparkline = Date.get_date_range(num_day)
|
||||||
|
sparklines_value = []
|
||||||
|
for date_day in date_range_sparkline:
|
||||||
|
nb_seen_this_day = r_serv_term.zscore('tracked_term:stat:{}'.format(term_uuid), date_day)
|
||||||
|
if nb_seen_this_day is None:
|
||||||
|
nb_seen_this_day = 0
|
||||||
|
sparklines_value.append(int(nb_seen_this_day))
|
||||||
|
return sparklines_value
|
||||||
|
|
||||||
|
def get_list_trackeed_term_tags(term_uuid):
|
||||||
|
res = r_serv_term.smembers('tracked_term:tags:{}'.format(term_uuid))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_list_trackeed_term_mails(term_uuid):
|
||||||
|
res = r_serv_term.smembers('tracked_term:mail:{}'.format(term_uuid))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_user_tracked_term_uuid(user_id):
|
||||||
|
return list(r_serv_term.smembers('user:tracked_term:{}'.format(user_id)))
|
||||||
|
|
||||||
|
def get_global_tracked_term_uuid():
|
||||||
|
return list(r_serv_term.smembers('global:tracked_term'))
|
||||||
|
|
||||||
|
def get_all_user_tracked_terms(user_id):
|
||||||
|
all_user_term = []
|
||||||
|
all_user_term_uuid = get_user_tracked_term_uuid(user_id)
|
||||||
|
|
||||||
|
for term_uuid in all_user_term_uuid:
|
||||||
|
all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True))
|
||||||
|
return all_user_term
|
||||||
|
|
||||||
|
def get_all_global_tracked_terms():
|
||||||
|
all_user_term = []
|
||||||
|
all_user_term_uuid = get_global_tracked_term_uuid()
|
||||||
|
|
||||||
|
for term_uuid in all_user_term_uuid:
|
||||||
|
all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True))
|
||||||
|
return all_user_term
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def get_global_tracked_term():
|
|
||||||
dict_tracked = {}
|
|
||||||
tracked_set = list(r_serv_term.smembers('global:TrackedSetSet'))
|
|
||||||
tracked_regex = list(r_serv_term.smembers('global:TrackedRegexSet'))
|
|
||||||
tracked_terms = list(r_serv_term.smembers('global:TrackedSetTermSet'))
|
|
||||||
return {'term': tracked_terms, 'set': tracked_terms, 'regex': tracked_regex}
|
|
||||||
|
|
||||||
def get_user_tracked_term(user_id):
|
|
||||||
dict_tracked = {}
|
|
||||||
tracked_set = list(r_serv_term.smembers('user:{}:TrackedSetSet'.format(user_id)))
|
|
||||||
tracked_regex = list(r_serv_term.smembers('user:{}:TrackedRegexSet').format(user_id))
|
|
||||||
tracked_terms = list(r_serv_term.smembers('user:{}:TrackedSetTermSet').format(user_id))
|
|
||||||
return {'term': tracked_terms, 'set': tracked_terms, 'regex': tracked_regex}
|
|
||||||
|
|
|
||||||
|
|
@ -708,7 +708,7 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### Delete term tracker: `api/v1/add/tracker/term/item`<a name="delete_term_tracker"></a>
|
### Delete term tracker: `api/v1/delete/tracker/term/item`<a name="delete_term_tracker"></a>
|
||||||
|
|
||||||
#### Description
|
#### Description
|
||||||
Delete term tracker
|
Delete term tracker
|
||||||
|
|
|
||||||
|
|
@ -6,20 +6,25 @@
|
||||||
|
|
||||||
note: The matching of credential against supplied credential is done using Levenshtein distance
|
note: The matching of credential against supplied credential is done using Levenshtein distance
|
||||||
'''
|
'''
|
||||||
|
import json
|
||||||
import redis
|
import redis
|
||||||
import datetime
|
import datetime
|
||||||
import calendar
|
import calendar
|
||||||
import flask
|
import flask
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
|
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
|
||||||
|
|
||||||
from Role_Manager import login_admin, login_analyst
|
from Role_Manager import login_admin, login_analyst
|
||||||
from flask_login import login_required
|
from flask_login import login_required, current_user
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import Paste
|
|
||||||
from pprint import pprint
|
from pprint import pprint
|
||||||
import Levenshtein
|
import Levenshtein
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------
|
||||||
|
|
||||||
|
import Paste
|
||||||
|
import Term
|
||||||
|
|
||||||
# ============ VARIABLES ============
|
# ============ VARIABLES ============
|
||||||
import Flask_config
|
import Flask_config
|
||||||
|
|
||||||
|
|
@ -146,337 +151,110 @@ def save_tag_to_auto_push(list_tag):
|
||||||
|
|
||||||
# ============ ROUTES ============
|
# ============ ROUTES ============
|
||||||
|
|
||||||
@terms.route("/terms_management/")
|
@terms.route("/tracker_term")
|
||||||
|
def tracked_term_menu():
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
user_term = Term.get_all_user_tracked_terms(user_id)
|
||||||
|
global_term = Term.get_all_global_tracked_terms()
|
||||||
|
return render_template("tracker_term_management.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
|
|
||||||
|
@terms.route("/tracker/add", methods=['GET', 'POST'])
|
||||||
@login_required
|
@login_required
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def terms_management():
|
def add_tracked_term_menu():
|
||||||
per_paste = request.args.get('per_paste')
|
if request.method == 'POST':
|
||||||
if per_paste == "1" or per_paste is None:
|
term = request.form.get("term")
|
||||||
per_paste_text = "per_paste_"
|
term_type = request.form.get("tracker_type")
|
||||||
per_paste = 1
|
nb_words = request.form.get("nb_word", 1)
|
||||||
else:
|
level = request.form.get("level", 1)
|
||||||
per_paste_text = ""
|
tags = request.form.get("tags", [])
|
||||||
per_paste = 0
|
mails = request.form.get("mails", [])
|
||||||
|
|
||||||
today = datetime.datetime.now()
|
if mails:
|
||||||
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
|
mails = mails.split()
|
||||||
today_timestamp = calendar.timegm(today.timetuple())
|
if tags:
|
||||||
|
tags = tags.split()
|
||||||
# Map tracking if notifications are enabled for a specific term
|
input_dict = {"term": term, "type": term_type, "nb_words": nb_words, "tags": tags, "mails": mails}
|
||||||
notificationEnabledDict = {}
|
user_id = current_user.get_id()
|
||||||
|
res = Term.parse_json_term_to_add(input_dict, user_id)
|
||||||
# Maps a specific term to the associated email addresses
|
if res[1] == 200:
|
||||||
notificationEMailTermMapping = {}
|
return redirect(url_for('terms.tracked_term_menu'))
|
||||||
notificationTagsTermMapping = {}
|
|
||||||
|
|
||||||
#Regex
|
|
||||||
trackReg_list = []
|
|
||||||
trackReg_list_values = []
|
|
||||||
trackReg_list_num_of_paste = []
|
|
||||||
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
|
|
||||||
|
|
||||||
notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
|
|
||||||
notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
|
|
||||||
|
|
||||||
if tracked_regex not in notificationEnabledDict:
|
|
||||||
notificationEnabledDict[tracked_regex] = False
|
|
||||||
|
|
||||||
trackReg_list.append(tracked_regex)
|
|
||||||
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
|
|
||||||
|
|
||||||
term_date = r_serv_term.hget(TrackedRegexDate_Name, tracked_regex)
|
|
||||||
|
|
||||||
set_paste_name = "regex_" + tracked_regex
|
|
||||||
trackReg_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
|
|
||||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
|
||||||
value_range.append(term_date)
|
|
||||||
trackReg_list_values.append(value_range)
|
|
||||||
|
|
||||||
if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
|
||||||
notificationEnabledDict[tracked_regex] = True
|
|
||||||
|
|
||||||
#Set
|
|
||||||
trackSet_list = []
|
|
||||||
trackSet_list_values = []
|
|
||||||
trackSet_list_num_of_paste = []
|
|
||||||
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
|
|
||||||
tracked_set = tracked_set
|
|
||||||
|
|
||||||
notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
|
|
||||||
notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
|
|
||||||
|
|
||||||
if tracked_set not in notificationEnabledDict:
|
|
||||||
notificationEnabledDict[tracked_set] = False
|
|
||||||
|
|
||||||
trackSet_list.append(tracked_set)
|
|
||||||
value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
|
|
||||||
|
|
||||||
term_date = r_serv_term.hget(TrackedSetDate_Name, tracked_set)
|
|
||||||
|
|
||||||
set_paste_name = "set_" + tracked_set
|
|
||||||
trackSet_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
|
|
||||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
|
||||||
value_range.append(term_date)
|
|
||||||
trackSet_list_values.append(value_range)
|
|
||||||
|
|
||||||
if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
|
||||||
notificationEnabledDict[tracked_set] = True
|
|
||||||
|
|
||||||
#Tracked terms
|
|
||||||
track_list = []
|
|
||||||
track_list_values = []
|
|
||||||
track_list_num_of_paste = []
|
|
||||||
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
|
|
||||||
|
|
||||||
notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
|
|
||||||
notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
|
|
||||||
|
|
||||||
if tracked_term not in notificationEnabledDict:
|
|
||||||
notificationEnabledDict[tracked_term] = False
|
|
||||||
|
|
||||||
track_list.append(tracked_term)
|
|
||||||
value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
|
|
||||||
|
|
||||||
term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
|
|
||||||
|
|
||||||
set_paste_name = "tracked_" + tracked_term
|
|
||||||
|
|
||||||
track_list_num_of_paste.append( r_serv_term.scard(set_paste_name) )
|
|
||||||
|
|
||||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
|
||||||
value_range.append(term_date)
|
|
||||||
track_list_values.append(value_range)
|
|
||||||
|
|
||||||
if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
|
||||||
notificationEnabledDict[tracked_term] = True
|
|
||||||
|
|
||||||
#blacklist terms
|
|
||||||
black_list = []
|
|
||||||
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
|
|
||||||
term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
|
|
||||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
|
||||||
black_list.append([blacked_term, term_date])
|
|
||||||
|
|
||||||
return render_template("terms_management.html",
|
|
||||||
black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
|
|
||||||
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
|
|
||||||
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
|
|
||||||
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
|
|
||||||
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
|
|
||||||
notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
|
|
||||||
|
|
||||||
|
|
||||||
@terms.route("/terms_management_query_paste/")
|
|
||||||
@login_required
|
|
||||||
@login_analyst
|
|
||||||
def terms_management_query_paste():
|
|
||||||
term = request.args.get('term')
|
|
||||||
paste_info = []
|
|
||||||
|
|
||||||
# check if regex or not
|
|
||||||
if term.startswith('/') and term.endswith('/'):
|
|
||||||
set_paste_name = "regex_" + term
|
|
||||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
|
||||||
elif term.startswith('\\') and term.endswith('\\'):
|
|
||||||
set_paste_name = "set_" + term
|
|
||||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
|
||||||
else:
|
|
||||||
set_paste_name = "tracked_" + term
|
|
||||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
|
||||||
|
|
||||||
for path in track_list_path:
|
|
||||||
paste = Paste.Paste(path)
|
|
||||||
p_date = str(paste._get_p_date())
|
|
||||||
p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8]
|
|
||||||
p_source = paste.p_source
|
|
||||||
p_size = paste.p_size
|
|
||||||
p_mime = paste.p_mime
|
|
||||||
p_lineinfo = paste.get_lines_info()
|
|
||||||
p_content = paste.get_p_content()
|
|
||||||
if p_content != 0:
|
|
||||||
p_content = p_content[0:400]
|
|
||||||
paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
|
|
||||||
|
|
||||||
return jsonify(paste_info)
|
|
||||||
|
|
||||||
|
|
||||||
@terms.route("/terms_management_query/")
|
|
||||||
@login_required
|
|
||||||
@login_analyst
|
|
||||||
def terms_management_query():
|
|
||||||
TrackedTermsDate_Name = "TrackedTermDate"
|
|
||||||
BlackListTermsDate_Name = "BlackListTermDate"
|
|
||||||
term = request.args.get('term')
|
|
||||||
section = request.args.get('section')
|
|
||||||
|
|
||||||
today = datetime.datetime.now()
|
|
||||||
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
today_timestamp = calendar.timegm(today.timetuple())
|
|
||||||
value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
|
|
||||||
|
|
||||||
if section == "followTerm":
|
|
||||||
term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
|
|
||||||
elif section == "blacklistTerm":
|
|
||||||
term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
|
|
||||||
|
|
||||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
|
||||||
value_range.append(str(term_date))
|
|
||||||
return jsonify(value_range)
|
|
||||||
|
|
||||||
|
|
||||||
@terms.route("/terms_management_action/", methods=['GET'])
|
|
||||||
@login_required
|
|
||||||
@login_analyst
|
|
||||||
def terms_management_action():
|
|
||||||
today = datetime.datetime.now()
|
|
||||||
today = today.replace(microsecond=0)
|
|
||||||
today_timestamp = calendar.timegm(today.timetuple())
|
|
||||||
|
|
||||||
|
|
||||||
section = request.args.get('section')
|
|
||||||
action = request.args.get('action')
|
|
||||||
term = request.args.get('term')
|
|
||||||
notificationEmailsParam = request.args.get('emailAddresses')
|
|
||||||
input_tags = request.args.get('tags')
|
|
||||||
|
|
||||||
if action is None or term is None or notificationEmailsParam is None:
|
|
||||||
return "None"
|
|
||||||
else:
|
|
||||||
if section == "followTerm":
|
|
||||||
if action == "add":
|
|
||||||
|
|
||||||
# Make a list of all passed email addresses
|
|
||||||
notificationEmails = notificationEmailsParam.split()
|
|
||||||
|
|
||||||
validNotificationEmails = []
|
|
||||||
# check for valid email addresses
|
|
||||||
for email in notificationEmails:
|
|
||||||
# Really basic validation:
|
|
||||||
# has exactly one @ sign, and at least one . in the part after the @
|
|
||||||
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
|
|
||||||
validNotificationEmails.append(email)
|
|
||||||
|
|
||||||
# create tags list
|
|
||||||
list_tags = input_tags.split()
|
|
||||||
|
|
||||||
# check if regex/set or simple term
|
|
||||||
#regex
|
|
||||||
if term.startswith('/') and term.endswith('/'):
|
|
||||||
r_serv_term.sadd(TrackedRegexSet_Name, term)
|
|
||||||
r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp)
|
|
||||||
# add all valid emails to the set
|
|
||||||
for email in validNotificationEmails:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
|
|
||||||
# enable notifications by default
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
|
|
||||||
# add tags list
|
|
||||||
for tag in list_tags:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
|
|
||||||
save_tag_to_auto_push(list_tags)
|
|
||||||
|
|
||||||
#set
|
|
||||||
elif term.startswith('\\') and term.endswith('\\'):
|
|
||||||
tab_term = term[1:-1]
|
|
||||||
perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_term)
|
|
||||||
if perc_finder is not None:
|
|
||||||
match_percent = perc_finder.group(0)[1:-1]
|
|
||||||
set_to_add = term
|
|
||||||
else:
|
|
||||||
match_percent = DEFAULT_MATCH_PERCENT
|
|
||||||
set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent)
|
|
||||||
r_serv_term.sadd(TrackedSetSet_Name, set_to_add)
|
|
||||||
r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp)
|
|
||||||
# add all valid emails to the set
|
|
||||||
for email in validNotificationEmails:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
|
|
||||||
# enable notifications by default
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
|
|
||||||
# add tags list
|
|
||||||
for tag in list_tags:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
|
|
||||||
save_tag_to_auto_push(list_tags)
|
|
||||||
|
|
||||||
#simple term
|
|
||||||
else:
|
|
||||||
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
|
|
||||||
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
|
|
||||||
# add all valid emails to the set
|
|
||||||
for email in validNotificationEmails:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
|
|
||||||
# enable notifications by default
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
|
|
||||||
# add tags list
|
|
||||||
for tag in list_tags:
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
|
|
||||||
save_tag_to_auto_push(list_tags)
|
|
||||||
|
|
||||||
elif action == "toggleEMailNotification":
|
|
||||||
# get the current state
|
|
||||||
if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
|
|
||||||
# remove it
|
|
||||||
r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower())
|
|
||||||
else:
|
|
||||||
# add it
|
|
||||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
|
|
||||||
|
|
||||||
#del action
|
|
||||||
else:
|
|
||||||
if term.startswith('/') and term.endswith('/'):
|
|
||||||
r_serv_term.srem(TrackedRegexSet_Name, term)
|
|
||||||
r_serv_term.hdel(TrackedRegexDate_Name, term)
|
|
||||||
elif term.startswith('\\') and term.endswith('\\'):
|
|
||||||
r_serv_term.srem(TrackedSetSet_Name, term)
|
|
||||||
r_serv_term.hdel(TrackedSetDate_Name, term)
|
|
||||||
else:
|
|
||||||
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
|
|
||||||
r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
|
|
||||||
|
|
||||||
# delete the associated notification emails too
|
|
||||||
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
|
|
||||||
# delete the associated tags set
|
|
||||||
r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
|
|
||||||
|
|
||||||
elif section == "blacklistTerm":
|
|
||||||
if action == "add":
|
|
||||||
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
|
|
||||||
r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
|
|
||||||
else:
|
|
||||||
r_serv_term.srem(BlackListTermsSet_Name, term.lower())
|
|
||||||
else:
|
else:
|
||||||
return "None"
|
## TODO: use modal
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
else:
|
||||||
|
return render_template("Add_tracker.html")
|
||||||
|
|
||||||
to_return = {}
|
@terms.route("/tracker/show_term_tracker")
|
||||||
to_return["section"] = section
|
|
||||||
to_return["action"] = action
|
|
||||||
to_return["term"] = term
|
|
||||||
return jsonify(to_return)
|
|
||||||
|
|
||||||
@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
|
|
||||||
@login_required
|
@login_required
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def delete_terms_tags():
|
def show_term_tracker():
|
||||||
term = request.form.get('term')
|
user_id = current_user.get_id()
|
||||||
tags_to_delete = request.form.getlist('tags_to_delete')
|
term_uuid = request.args.get('uuid', None)
|
||||||
|
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||||
|
if res: # invalid access
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
if term is not None and tags_to_delete is not None:
|
date_from = request.args.get('date_from')
|
||||||
for tag in tags_to_delete:
|
date_to = request.args.get('date_to')
|
||||||
r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
|
|
||||||
return redirect(url_for('terms.terms_management'))
|
if date_from:
|
||||||
|
date_from = date_from.replace('-', '')
|
||||||
|
if date_to:
|
||||||
|
date_to = date_to.replace('-', '')
|
||||||
|
|
||||||
|
term_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, tags=True, mails=True, sparkline=True)
|
||||||
|
|
||||||
|
if date_from:
|
||||||
|
res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
||||||
|
if res[1] !=200:
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
term_metadata['items'] = res[0]['items']
|
||||||
|
term_metadata['date_from'] = res[0]['date_from']
|
||||||
|
term_metadata['date_to'] = res[0]['date_to']
|
||||||
else:
|
else:
|
||||||
return 'None args', 400
|
term_metadata['items'] = []
|
||||||
|
term_metadata['date_from'] = ''
|
||||||
|
term_metadata['date_to'] = ''
|
||||||
|
|
||||||
@terms.route("/terms_management/delete_terms_email", methods=['GET'])
|
return render_template("showTrackerTerm.html", term_metadata=term_metadata, bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
|
@terms.route("/tracker/update_tracker_tags", methods=['POST'])
|
||||||
@login_required
|
@login_required
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def delete_terms_email():
|
def update_tracker_tags():
|
||||||
term = request.args.get('term')
|
user_id = current_user.get_id()
|
||||||
email = request.args.get('email')
|
term_uuid = request.form.get('uuid')
|
||||||
|
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||||
if term is not None and email is not None:
|
if res: # invalid access
|
||||||
r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
return redirect(url_for('terms.terms_management'))
|
tags = request.form.get('tags')
|
||||||
|
if tags:
|
||||||
|
tags = tags.split()
|
||||||
else:
|
else:
|
||||||
return 'None args', 400
|
tags = []
|
||||||
|
Term.replace_tracked_term_tags(term_uuid, tags)
|
||||||
|
return redirect(url_for('terms.show_term_tracker', uuid=term_uuid))
|
||||||
|
|
||||||
|
@terms.route("/tracker/update_tracker_mails", methods=['POST'])
|
||||||
|
@login_required
|
||||||
|
@login_analyst
|
||||||
|
def update_tracker_mails():
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
term_uuid = request.form.get('uuid')
|
||||||
|
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
|
||||||
|
if res: # invalid access
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
mails = request.form.get('mails')
|
||||||
|
if mails:
|
||||||
|
mails = mails.split()
|
||||||
|
else:
|
||||||
|
mails = []
|
||||||
|
res = Term.replace_tracked_term_mails(term_uuid, mails)
|
||||||
|
if res: # invalid mail
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
return redirect(url_for('terms.show_term_tracker', uuid=term_uuid))
|
||||||
|
|
||||||
|
|
||||||
@terms.route("/terms_plot_tool/")
|
@terms.route("/terms_plot_tool/")
|
||||||
|
|
|
||||||
153
var/www/modules/terms/templates/Add_tracker.html
Normal file
153
var/www/modules/terms/templates/Add_tracker.html
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>AIL-Framework</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'crawler/menu_sidebar.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card mb-3 mt-1">
|
||||||
|
<div class="card-header">
|
||||||
|
<h5 class="card-title">Create a new tracker</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
|
||||||
|
|
||||||
|
<form action="{{ url_for('terms.add_tracked_term_menu') }}" method='post'>
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-xl-9">
|
||||||
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
|
<div class="input-group-prepend">
|
||||||
|
<div class="input-group-text"><i class="fas fa-tag"></i></div>
|
||||||
|
</div>
|
||||||
|
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text">
|
||||||
|
</div>
|
||||||
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
|
<div class="input-group-prepend">
|
||||||
|
<div class="input-group-text"><i class="fas fa-at"></i></div>
|
||||||
|
</div>
|
||||||
|
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-12 col-xl-3">
|
||||||
|
<div class="custom-control custom-switch mt-1">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="level" id="id_level" checked>
|
||||||
|
<label class="custom-control-label" for="id_level">
|
||||||
|
<i class="fas fa-users"></i> Show tracker to all Users
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
|
||||||
|
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
|
||||||
|
<option disabled selected value> -- Select a tracker type -- </option>
|
||||||
|
<option value="word">Word</option>
|
||||||
|
<option value="set">Set</option>
|
||||||
|
<option value="regex">Regex</option>
|
||||||
|
</select>
|
||||||
|
|
||||||
|
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-lg-10">
|
||||||
|
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
|
||||||
|
</div>
|
||||||
|
<div class="col-12 col-lg-2">
|
||||||
|
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<br>
|
||||||
|
<button class="btn btn-success mt-2">
|
||||||
|
<i class="fas fa-plus"></i> Add Tracker
|
||||||
|
</button>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var chart = {};
|
||||||
|
$(document).ready(function(){
|
||||||
|
$("#page-Crawler").addClass("active");
|
||||||
|
$("#nav_manual_crawler").addClass("active");
|
||||||
|
$("#tracker_desc").hide();
|
||||||
|
$("#term").hide();
|
||||||
|
$("#nb_word").hide();
|
||||||
|
|
||||||
|
$('#tracker_type').on('change', function() {
|
||||||
|
var tracker_type = this.value;
|
||||||
|
if (tracker_type=="word") {
|
||||||
|
$("#tracker_desc").text("Term to track:");
|
||||||
|
$("#tracker_desc").show();
|
||||||
|
$("#term").show();
|
||||||
|
$("#nb_word").hide();
|
||||||
|
} else if (tracker_type=="set") {
|
||||||
|
$("#tracker_desc").text("Terms to track (space separated), explain nb");
|
||||||
|
$("#tracker_desc").show();
|
||||||
|
$("#term").show();
|
||||||
|
$("#nb_word").show();
|
||||||
|
} else {
|
||||||
|
$("#tracker_desc").text("valid python regex");
|
||||||
|
$("#tracker_desc").show();
|
||||||
|
$("#term").show();
|
||||||
|
$("#nb_word").hide();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
</script>
|
||||||
319
var/www/modules/terms/templates/showTrackerTerm.html
Normal file
319
var/www/modules/terms/templates/showTrackerTerm.html
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
|
||||||
|
<title>AIL Framework - AIL</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.line_sparkline {
|
||||||
|
fill: none;
|
||||||
|
stroke: #000;
|
||||||
|
stroke-width: 2.0px;
|
||||||
|
}
|
||||||
|
.btn-link {
|
||||||
|
color: #000000
|
||||||
|
}
|
||||||
|
.mouse_pointer{
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'tracker/menu_sidebar.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||||
|
<h4 class="text-secondary">{{ term_metadata['uuid'] }} </h4>
|
||||||
|
<ul class="list-group mb-2">
|
||||||
|
<li class="list-group-item py-0">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-10">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Tracked</th>
|
||||||
|
<th>Date added</th>
|
||||||
|
<th>Level</th>
|
||||||
|
<th>Created by</th>
|
||||||
|
<th>First seen</th>
|
||||||
|
<th>Last seen</th>
|
||||||
|
<th>Tags <span class="btn-link btn-interaction mouse_pointer" title="Edit Tags List" onclick="edit_tags();"><i class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||||
|
<th>Email <span class="btn-link btn-interaction mouse_pointer" title="Edit Email List" onclick="edit_mails();"><i class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>{{ term_metadata['type'] }}</td>
|
||||||
|
<td>{{ term_metadata['term'] }}</td>
|
||||||
|
<td>{{ term_metadata['date'][0:4] }}/{{ term_metadata['date'][4:6] }}/{{ term_metadata['date'][6:8] }}</td>
|
||||||
|
<td>{{ term_metadata['level'] }}</td>
|
||||||
|
<td>{{ term_metadata['user_id'] }}</td>
|
||||||
|
<td>
|
||||||
|
{% if term_metadata['first_seen'] %}
|
||||||
|
{{ term_metadata['first_seen'][0:4] }}/{{ term_metadata['first_seen'][4:6] }}/{{ term_metadata['first_seen'][6:8] }}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if term_metadata['last_seen'] %}
|
||||||
|
{{ term_metadata['last_seen'][0:4] }}/{{ term_metadata['last_seen'][4:6] }}/{{ term_metadata['last_seen'][6:8] }}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for tag in term_metadata['tags'] %}
|
||||||
|
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for mail in term_metadata['mails'] %}
|
||||||
|
{{ mail }}<br>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-1">
|
||||||
|
<div id="sparkline"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<div id="div_edit_tags">
|
||||||
|
<form action="{{ url_for('terms.update_tracker_tags') }}" method='post'>
|
||||||
|
<input name="uuid" type="text" value="{{term_metadata['uuid']}}" hidden>
|
||||||
|
<div>All Tags added for this tracker, space separated: </div>
|
||||||
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
|
<div class="input-group-prepend">
|
||||||
|
<div class="input-group-text"><i class="fas fa-tag"></i></div>
|
||||||
|
</div>
|
||||||
|
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text"
|
||||||
|
value="{% for tag in term_metadata['tags'] %}{{tag}} {% endfor %}">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button class="btn btn-info">
|
||||||
|
<i class="fas fa-pencil-alt"></i> Edit Tags
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="div_edit_mails">
|
||||||
|
<form action="{{ url_for('terms.update_tracker_mails') }}" method='post'>
|
||||||
|
<input name="uuid" type="text" value="{{term_metadata['uuid']}}" hidden>
|
||||||
|
<div>All E-Mails to Notify for this tracker, space separated: </div>
|
||||||
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
|
<div class="input-group-prepend">
|
||||||
|
<div class="input-group-text"><i class="fas fa-at"></i></div>
|
||||||
|
</div>
|
||||||
|
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"
|
||||||
|
value="{% for mail in term_metadata['mails'] %}{{mail}} {% endfor %}">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button class="btn btn-info">
|
||||||
|
<i class="fas fa-pencil-alt"></i> Edit Email Notification
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{hash}}" target="blank" class="float-right" style="font-size: 15px">
|
||||||
|
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i>
|
||||||
|
</button>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="card mb-3 mt-1">
|
||||||
|
<div class="card-body">
|
||||||
|
|
||||||
|
<div class="row mb-3">
|
||||||
|
<div class="col-md-6">
|
||||||
|
<div class="input-group" id="date-range-from">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ term_metadata['date_from'] }}" name="date_from" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<div class="input-group" id="date-range-to">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ term_metadata['date_to'] }}" name="date_to" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
|
||||||
|
<i class="fas fa-search"></i> Search Tracked Items
|
||||||
|
</button>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{%if term_metadata['items']%}
|
||||||
|
<table class="table table-bordered table-hover" id="myTable_">
|
||||||
|
<thead class="thead-dark">
|
||||||
|
<tr>
|
||||||
|
<th>Item</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
|
||||||
|
{% for item in term_metadata['items'] %}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{item}}">
|
||||||
|
<div style="line-height:0.9;">{{ item }}</div>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#div_edit_mails').hide();
|
||||||
|
$('#div_edit_tags').hide();
|
||||||
|
$("#page-Decoded").addClass("active");
|
||||||
|
|
||||||
|
$('#date-range-from').dateRangePicker({
|
||||||
|
separator : ' to ',
|
||||||
|
getValue: function(){
|
||||||
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
|
else
|
||||||
|
return '';
|
||||||
|
},
|
||||||
|
setValue: function(s,s1,s2){
|
||||||
|
$('#date-range-from-input').val(s1);
|
||||||
|
$('#date-range-to-input').val(s2);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
$('#date-range-to').dateRangePicker({
|
||||||
|
separator : ' to ',
|
||||||
|
getValue: function(){
|
||||||
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
|
else
|
||||||
|
return '';
|
||||||
|
},
|
||||||
|
setValue: function(s,s1,s2){
|
||||||
|
$('#date-range-from-input').val(s1);
|
||||||
|
$('#date-range-to-input').val(s2);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
sparklines("sparkline", {{ term_metadata['sparkline'] }});
|
||||||
|
});
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function edit_tags(){
|
||||||
|
$('#div_edit_mails').hide();
|
||||||
|
$('#div_edit_tags').show();
|
||||||
|
}
|
||||||
|
|
||||||
|
function edit_mails(){
|
||||||
|
$('#div_edit_tags').hide();
|
||||||
|
$('#div_edit_mails').show();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getItems() {
|
||||||
|
var date_from = $('#date-range-from-input').val();
|
||||||
|
var date_to =$('#date-range-to-input').val();
|
||||||
|
window.location.replace("{{ url_for('terms.show_term_tracker') }}?uuid={{ term_metadata['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
|
||||||
|
}
|
||||||
|
|
||||||
|
</script>
|
||||||
|
<script>
|
||||||
|
//var data = [6,3,3,2,5,3,9];
|
||||||
|
|
||||||
|
// a sparklines plot
|
||||||
|
function sparklines(id, points) {
|
||||||
|
var width_spark = 100, height_spark = 60;
|
||||||
|
|
||||||
|
var data = []
|
||||||
|
for (i = 0; i < points.length; i++) {
|
||||||
|
data[i] = {
|
||||||
|
'x': i,
|
||||||
|
'y': +points[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var x = d3.scaleLinear()
|
||||||
|
.range([0, width_spark - 10])
|
||||||
|
.domain([0,5]);
|
||||||
|
|
||||||
|
var y = d3.scaleLinear()
|
||||||
|
.range([height_spark, 0])
|
||||||
|
.domain([0,10]);
|
||||||
|
|
||||||
|
var line = d3.line()
|
||||||
|
.x(function(d) {return x(d.x)})
|
||||||
|
.y(function(d) {return y(d.y)});
|
||||||
|
|
||||||
|
d3.select("#"+id).append('svg')
|
||||||
|
.attr('width', width_spark)
|
||||||
|
.attr('height', height_spark)
|
||||||
|
.append('path')
|
||||||
|
.attr('class','line_sparkline')
|
||||||
|
.datum(data)
|
||||||
|
.attr('d', line);
|
||||||
|
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
181
var/www/modules/terms/templates/tracker_term_management.html
Normal file
181
var/www/modules/terms/templates/tracker_term_management.html
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
|
||||||
|
<title>Terms Management</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
|
||||||
|
.btn-link {
|
||||||
|
color: #000000
|
||||||
|
}
|
||||||
|
.mouse_pointer{
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.lb-md {
|
||||||
|
font-size: 16px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'tracker/menu_sidebar.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-header">
|
||||||
|
<h5 class="card-title">Your Tracked Terms</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<table id="table_user_terms" class="table table-striped table-bordered">
|
||||||
|
<thead class="bg-dark text-white">
|
||||||
|
<tr>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Tracked Term</th>
|
||||||
|
<th>First seen</th>
|
||||||
|
<th>Last seen</th>
|
||||||
|
<th>Email notification</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody style="font-size: 15px;">
|
||||||
|
{% for dict_uuid in user_term %}
|
||||||
|
<tr>
|
||||||
|
<td>{{dict_uuid['type']}}</td>
|
||||||
|
<td>
|
||||||
|
<span><a target="_blank" href="{{ url_for('terms.show_term_tracker') }}?uuid={{ dict_uuid['uuid'] }}">{{dict_uuid['term']}}</a></span>
|
||||||
|
<div>
|
||||||
|
{% for tag in dict_uuid['tags'] %}
|
||||||
|
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||||
|
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if dict_uuid['first_seen'] %}
|
||||||
|
{{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if dict_uuid['last_seen'] %}
|
||||||
|
{{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for mail in dict_uuid['mails'] %}
|
||||||
|
{{ mail }}<br>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-header">
|
||||||
|
<h5 class="card-title">Global Tracked Terms</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<table id="table_global_terms" class="table table-striped table-bordered">
|
||||||
|
<thead class="bg-dark text-white">
|
||||||
|
<tr>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Tracked Term</th>
|
||||||
|
<th>First seen</th>
|
||||||
|
<th>Last seen</th>
|
||||||
|
<th>Email notification</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody style="font-size: 15px;">
|
||||||
|
{% for dict_uuid in global_term %}
|
||||||
|
<tr>
|
||||||
|
<td>{{dict_uuid['type']}}</td>
|
||||||
|
<td>
|
||||||
|
<span><a target="_blank" href="{{ url_for('terms.show_term_tracker') }}?uuid={{ dict_uuid['uuid'] }}">{{dict_uuid['term']}}</a></span>
|
||||||
|
<div>
|
||||||
|
{% for tag in dict_uuid['tags'] %}
|
||||||
|
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if dict_uuid['first_seen'] %}
|
||||||
|
{{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if dict_uuid['last_seen'] %}
|
||||||
|
{{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for mail in dict_uuid['mails'] %}
|
||||||
|
{{ mail }}<br>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#table_user_terms').DataTable({
|
||||||
|
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||||
|
"iDisplayLength": 10,
|
||||||
|
"order": [[ 0, "desc" ]]
|
||||||
|
});
|
||||||
|
$('#table_global_terms').DataTable({
|
||||||
|
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||||
|
"iDisplayLength": 10,
|
||||||
|
"order": [[ 0, "desc" ]]
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -19,7 +19,7 @@
|
||||||
<a class="nav-link" id="page-Browse-Items" href="{{ url_for('Tags.Tags_page') }}" aria-disabled="true"><i class="fas fa-tag"></i> Browse Items</a>
|
<a class="nav-link" id="page-Browse-Items" href="{{ url_for('Tags.Tags_page') }}" aria-disabled="true"><i class="fas fa-tag"></i> Browse Items</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item mr-3">
|
<li class="nav-item mr-3">
|
||||||
<a class="nav-link" href="{{ url_for('terms.terms_management') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
|
<a class="nav-link" id="page-Tracker" href="{{ url_for('terms.tracked_term_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item mr-3">
|
<li class="nav-item mr-3">
|
||||||
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
||||||
|
|
|
||||||
24
var/www/templates/tracker/menu_sidebar.html
Normal file
24
var/www/templates/tracker/menu_sidebar.html
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
|
||||||
|
|
||||||
|
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
|
||||||
|
<i class="fas fa-align-left"></i>
|
||||||
|
<span>Toggle Sidebar</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
|
||||||
|
<h5 class="d-flex text-muted w-100">
|
||||||
|
<span>Terms Tracker </span>
|
||||||
|
<a class="ml-auto" href="{{url_for('terms.add_tracked_term_menu')}}">
|
||||||
|
<i class="fas fa-plus-circle ml-auto"></i>
|
||||||
|
</a>
|
||||||
|
</h5>
|
||||||
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('terms.tracked_term_menu')}}" id="nav_dashboard">
|
||||||
|
<i class="fas fa-search"></i>
|
||||||
|
<span>Tracked Terms</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
Loading…
Reference in a new issue