mirror of
https://github.com/ail-project/ail-framework.git
synced 2025-09-09 08:32:41 +00:00
New feature in Terms: regex and set of words support (draft)
This commit is contained in:
Mokaddem
parent
be93af6f5a
commit
2da4c572c7
8 changed files with 366 additions and 22 deletions
|
|
@ -9,7 +9,7 @@ import datetime
|
|||
import calendar
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request
|
||||
|
||||
import re
|
||||
import Paste
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
|
@ -18,6 +18,22 @@ import Flask_config
|
|||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
|
||||
DEFAULT_MATCH_PERCENT = 50
|
||||
|
||||
#tracked
|
||||
TrackedTermsSet_Name = "TrackedSetTermSet"
|
||||
TrackedTermsDate_Name = "TrackedTermDate"
|
||||
#black
|
||||
BlackListTermsDate_Name = "BlackListTermDate"
|
||||
BlackListTermsSet_Name = "BlackListSetTermSet"
|
||||
#regex
|
||||
TrackedRegexSet_Name = "TrackedRegexSet"
|
||||
TrackedRegexDate_Name = "TrackedRegexDate"
|
||||
#set
|
||||
TrackedSetSet_Name = "TrackedSetSet"
|
||||
TrackedSetDate_Name = "TrackedSetDate"
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
||||
|
|
@ -47,15 +63,43 @@ def terms_management():
|
|||
per_paste_text = ""
|
||||
per_paste = 0
|
||||
|
||||
TrackedTermsSet_Name = "TrackedSetTermSet"
|
||||
BlackListTermsSet_Name = "BlackListSetTermSet"
|
||||
TrackedTermsDate_Name = "TrackedTermDate"
|
||||
BlackListTermsDate_Name = "BlackListTermDate"
|
||||
|
||||
today = datetime.datetime.now()
|
||||
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_timestamp = calendar.timegm(today.timetuple())
|
||||
|
||||
#Regex
|
||||
trackReg_list = []
|
||||
trackReg_list_values = []
|
||||
trackReg_list_num_of_paste = []
|
||||
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
|
||||
trackReg_list.append(tracked_regex)
|
||||
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
|
||||
|
||||
term_date = r_serv_term.hget(TrackedRegexDate_Name, tracked_regex)
|
||||
|
||||
set_paste_name = "regex_" + tracked_regex
|
||||
trackReg_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
|
||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
value_range.append(term_date)
|
||||
trackReg_list_values.append(value_range)
|
||||
|
||||
#Set
|
||||
trackSet_list = []
|
||||
trackSet_list_values = []
|
||||
trackSet_list_num_of_paste = []
|
||||
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
|
||||
trackSet_list.append(tracked_set)
|
||||
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
|
||||
|
||||
term_date = r_serv_term.hget(TrackedSetDate_Name, tracked_set)
|
||||
|
||||
set_paste_name = "set_" + tracked_set
|
||||
trackSet_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
|
||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
value_range.append(term_date)
|
||||
trackSet_list_values.append(value_range)
|
||||
|
||||
#Tracked terms
|
||||
track_list = []
|
||||
track_list_values = []
|
||||
track_list_num_of_paste = []
|
||||
|
|
@ -72,23 +116,36 @@ def terms_management():
|
|||
track_list_values.append(value_range)
|
||||
|
||||
|
||||
#blacklist terms
|
||||
black_list = []
|
||||
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
|
||||
term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
|
||||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
black_list.append([blacked_term, term_date])
|
||||
|
||||
return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, per_paste=per_paste)
|
||||
return render_template("terms_management.html",
|
||||
black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
|
||||
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
|
||||
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
|
||||
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
|
||||
per_paste=per_paste)
|
||||
|
||||
|
||||
@app.route("/terms_management_query_paste/")
|
||||
def terms_management_query_paste():
|
||||
term = request.args.get('term')
|
||||
TrackedTermsSet_Name = "TrackedSetTermSet"
|
||||
paste_info = []
|
||||
|
||||
set_paste_name = "tracked_" + term
|
||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
||||
# check if regex or not
|
||||
if term.startswith('/') and term.endswith('/'):
|
||||
set_paste_name = "regex_" + term
|
||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
||||
elif term.startswith('\\') and term.endswith('\\'):
|
||||
set_paste_name = "set_" + term
|
||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
||||
else:
|
||||
set_paste_name = "tracked_" + term
|
||||
track_list_path = r_serv_term.smembers(set_paste_name)
|
||||
|
||||
for path in track_list_path:
|
||||
paste = Paste.Paste(path)
|
||||
|
|
@ -131,11 +188,6 @@ def terms_management_query():
|
|||
|
||||
@app.route("/terms_management_action/", methods=['GET'])
|
||||
def terms_management_action():
|
||||
TrackedTermsSet_Name = "TrackedSetTermSet"
|
||||
TrackedTermsDate_Name = "TrackedTermDate"
|
||||
BlackListTermsDate_Name = "BlackListTermDate"
|
||||
BlackListTermsSet_Name = "BlackListSetTermSet"
|
||||
|
||||
today = datetime.datetime.now()
|
||||
today = today.replace(microsecond=0)
|
||||
today_timestamp = calendar.timegm(today.timetuple())
|
||||
|
|
@ -149,10 +201,42 @@ def terms_management_action():
|
|||
else:
|
||||
if section == "followTerm":
|
||||
if action == "add":
|
||||
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
|
||||
r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp)
|
||||
# check if regex/set or simple term
|
||||
#regex
|
||||
if term.startswith('/') and term.endswith('/'):
|
||||
r_serv_term.sadd(TrackedRegexSet_Name, term)
|
||||
r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp)
|
||||
|
||||
#set
|
||||
elif term.startswith('\\') and term.endswith('\\'):
|
||||
tab_term = term[1:-1]
|
||||
perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_term)
|
||||
if perc_finder is not None:
|
||||
match_percent = perc_finder.group(0)[1:-1]
|
||||
set_to_add = term
|
||||
else:
|
||||
match_percent = DEFAULT_MATCH_PERCENT
|
||||
set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent)
|
||||
r_serv_term.sadd(TrackedSetSet_Name, set_to_add)
|
||||
r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp)
|
||||
|
||||
#simple term
|
||||
else:
|
||||
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
|
||||
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
|
||||
#del action
|
||||
else:
|
||||
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
|
||||
if term.startswith('/') and term.endswith('/'):
|
||||
r_serv_term.srem(TrackedRegexSet_Name, term)
|
||||
r_serv_term.hdel(TrackedRegexDate_Name, term)
|
||||
elif term.startswith('\\') and term.endswith('\\'):
|
||||
r_serv_term.srem(TrackedSetSet_Name, term)
|
||||
print(term)
|
||||
r_serv_term.hdel(TrackedSetDate_Name, term)
|
||||
else:
|
||||
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
|
||||
r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
|
||||
|
||||
elif section == "blacklistTerm":
|
||||
if action == "add":
|
||||
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
|
||||
|
|
|
|||
|
|
@ -105,9 +105,16 @@
|
|||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<div style="margin-bottom: 10px;">
|
||||
<table>
|
||||
<tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
|
||||
<tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
|
||||
<tr><td> - To set a custom matching <b>threshold</b> (defaut=50), append it at the end as a inner list '<b>[thresh]</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ..., [75]]\</b></td></tr>
|
||||
</table>
|
||||
</div>
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
|
||||
<input id="followTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
|
||||
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
|
||||
</div>
|
||||
|
||||
|
|
@ -124,6 +131,37 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!-- SET -->
|
||||
{% for set in trackSet_list %}
|
||||
<tr style="background-color: #cdffca;">
|
||||
<td>{{ set }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
</p></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
<!-- REGEX -->
|
||||
{% for regex in trackReg_list %}
|
||||
<tr style="background-color: #fffdca;">
|
||||
<td>{{ regex }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
</p></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
<!-- Normal term -->
|
||||
{% for term in track_list %}
|
||||
<tr>
|
||||
<td>{{ term }}</td>
|
||||
|
|
@ -318,9 +356,9 @@ function perform_operation(){
|
|||
var curr_section = $(this).attr('data-section');
|
||||
var curr_action = $(this).attr('data-action');
|
||||
if (curr_action == "add") {
|
||||
var curr_term = $('#'+curr_section+'Input').val().toLowerCase();;
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
} else {
|
||||
var curr_term = $(this).attr('data-content').toLowerCase();;
|
||||
var curr_term = $(this).attr('data-content');
|
||||
}
|
||||
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
|
||||
|
||||
|
|
@ -341,6 +379,7 @@ function perform_operation(){
|
|||
} else if (json.action == "delete") {
|
||||
// Find indexes of row which have the term in the first column
|
||||
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
|
||||
console.log(table_track.cell( rowIdx, 0 ).data())
|
||||
return table_track.cell( rowIdx, 0 ).data() === json.term;
|
||||
} );
|
||||
table_track.rows(index).remove().draw( false );
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue