diff --git a/OVERVIEW.md b/OVERVIEW.md index effb387d..3d3a62ab 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -26,6 +26,24 @@ ARDB overview ARDB_DB * DB 1 - Curve * DB 2 - TermFreq + ----------------------------------------- TERM ---------------------------------------- + + SET - 'TrackedRegexSet' term + + HSET - 'TrackedRegexDate' tracked_regex today_timestamp + + SET - 'TrackedSetSet' set_to_add + + HSET - 'TrackedSetDate' set_to_add today_timestamp + + SET - 'TrackedSetTermSet' term + + HSET - 'TrackedTermDate' tracked_regex today_timestamp + + SET - 'TrackedNotificationEmails_'+term/set email + + SET - 'TrackedNotifications' term/set + * DB 3 - Trending * DB 4 - Sentiment * DB 5 - TermCred diff --git a/bin/Curve.py b/bin/Curve.py index 8e228039..c7083c54 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -71,6 +73,11 @@ def check_if_tracked_term(term, path): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term): + msg = '{};{}'.format(tag, path) + p.populate_set_out(msg, 'Tags') + def getValueOverRange(word, startDate, num_day): to_return = 0 diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py index fae7a03a..0db7f2ee 100755 --- a/bin/RegexForTermsFrequency.py +++ b/bin/RegexForTermsFrequency.py @@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -129,6 +131,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + set_name = 'regex_' + dico_regexname_to_redis[regex_str] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py index 78de9b08..19ed7210 100755 --- a/bin/SetForTermsFrequency.py +++ b/bin/SetForTermsFrequency.py @@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -121,6 +123,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + print(the_set, "matched in", filename) set_name = 'set_' + dico_setname_to_redis[the_set] new_to_the_set = server_term.sadd(set_name, filename) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index deb5a069..0dc40448 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -32,13 +32,15 @@ publish = Redis_Words [Curve] subscribe = Redis_Words -publish = Redis_CurveManageTopSets +publish = Redis_CurveManageTopSets,Redis_Tags [RegexForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [SetForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [CurveManageTopSets] subscribe = Redis_CurveManageTopSets diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index b0794593..f489498d 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -24,6 +24,7 @@ cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred +bootstrap_label = Flask_config.bootstrap_label terms = Blueprint('terms', __name__, template_folder='templates') @@ -51,6 +52,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # same value as in `bin/NotificationHelper.py` # Keys will be e.g. TrackedNotificationEmails_ TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' @@ -152,6 +154,7 @@ def terms_management(): # Maps a specific term to the associated email addresses notificationEMailTermMapping = {} + notificationTagsTermMapping = {} #Regex trackReg_list = [] @@ -160,6 +163,7 @@ def terms_management(): for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) + notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) if tracked_regex not in notificationEnabledDict: notificationEnabledDict[tracked_regex] = False @@ -186,7 +190,7 @@ def terms_management(): tracked_set = tracked_set notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) - + notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) if tracked_set not in notificationEnabledDict: notificationEnabledDict[tracked_set] = False @@ -212,6 +216,7 @@ def terms_management(): for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) if tracked_term not in notificationEnabledDict: notificationEnabledDict[tracked_term] = False @@ -239,12 +244,14 @@ def terms_management(): term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" black_list.append([blacked_term, term_date]) + print(notificationTagsTermMapping) return render_template("terms_management.html", black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) + per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label, + notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping) @terms.route("/terms_management_query_paste/") @@ -313,6 +320,7 @@ def terms_management_action(): action = request.args.get('action') term = request.args.get('term') notificationEmailsParam = request.args.get('emailAddresses') + input_tags = request.args.get('tags') if action is None or term is None or notificationEmailsParam is None: return "None" @@ -334,6 +342,8 @@ def terms_management_action(): if re.match(r"[^@]+@[^@]+\.[^@]+", email): validNotificationEmails.append(email) + # create tags list + list_tags = input_tags.split() # check if regex/set or simple term #regex @@ -345,6 +355,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) #set elif term.startswith('\\') and term.endswith('\\'): @@ -363,6 +376,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) #simple term else: @@ -373,6 +389,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) elif action == "toggleEMailNotification": # get the current state @@ -397,6 +416,8 @@ def terms_management_action(): # delete the associated notification emails too r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) + # delete the associated tags set + r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term) elif section == "blacklistTerm": if action == "add": diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 0efda575..fc92946a 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -98,7 +98,8 @@
- + +
@@ -119,7 +120,16 @@ {% for set in trackSet_list %} - {{ set }} + + {{ set }} +
+ {% for tag in notificationTagsTermMapping[set] %} + + {{ tag }} + + {% endfor %} +
+ {{ trackSet_list_values[loop.index0][3] }} {{ trackSet_list_values[loop.index0][0] }} {{ trackSet_list_values[loop.index0][1] }} @@ -136,7 +146,16 @@ {% for regex in trackReg_list %} - {{ regex }} + + {{ regex }} +
+ {% for tag in notificationTagsTermMapping[regex] %} + + {{ tag }} + + {% endfor %} +
+ {{ trackReg_list_values[loop.index0][3] }} {{ trackReg_list_values[loop.index0][0] }} {{ trackReg_list_values[loop.index0][1] }} @@ -153,7 +172,16 @@ {% for term in track_list %} - {{ term }} + + {{ term }} +
+ {% for tag in notificationTagsTermMapping[term] %} + + {{ tag }} + + {% endfor %} +
+ {{ track_list_values[loop.index0][3] }} {{ track_list_values[loop.index0][0] }} {{ track_list_values[loop.index0][1] }} @@ -351,17 +379,19 @@ function perform_binding() { function perform_operation(){ var curr_section = $(this).attr('data-section'); var curr_action = $(this).attr('data-action'); + var row_tr = $(this).closest("tr"); if (curr_action == "add") { var curr_term = $('#'+curr_section+'Input').val(); var email_addresses = $('#followTermEMailNotificationReceiversInput').val(); + var tags = $('#followTermTag').val(); } else { var curr_term = $(this).attr('data-content'); var email_addresses = ""; } - var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses}; + var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags}; if (curr_term != "") { - console.log(data_to_send); + //console.log(data_to_send); $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){ if(status == "success") { var json = data; @@ -372,13 +402,8 @@ function perform_operation(){ $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ reload_per_paste(); }); - } else if (json.action == "delete") { - // Find indexes of row which have the term in the first column - var index = table_track.rows().eq( 0 ).filter( function (rowIdx) { - console.log(table_track.cell( rowIdx, 0 ).data()) - return table_track.cell( rowIdx, 0 ).data() === json.term; - } ); - table_track.rows(index).remove().draw( false ); + } else if (json.action == "delete") { + row_tr.remove() } } else if(json.section == "blacklistTerm"){ if(json.action == "add") {