chg: [Terms] tag tracked terms

OVERVIEW.md

@@ -26,6 +26,24 @@ ARDB overview
 ARDB_DB
 * DB 1 - Curve
 * DB 2 - TermFreq
+	----------------------------------------- TERM ----------------------------------------
+
+	SET - 'TrackedRegexSet'				term
+
+	HSET - 'TrackedRegexDate'			tracked_regex		today_timestamp
+
+	SET - 'TrackedSetSet'				set_to_add
+
+	HSET - 'TrackedSetDate'				set_to_add		today_timestamp
+
+	SET - 'TrackedSetTermSet'			term
+
+	HSET - 'TrackedTermDate'			tracked_regex		today_timestamp
+
+	SET - 'TrackedNotificationEmails_'+term/set	email
+
+	SET - 'TrackedNotifications'			term/set
+
 * DB 3 - Trending
 * DB 4 - Sentiment
 * DB 5 - TermCred

bin/Curve.py

@@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
             for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
                 sendEmailNotification(email, 'Term', mail_body)
 
+        # tag paste
+        for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
+            msg = '{};{}'.format(tag, path)
+            p.populate_set_out(msg, 'Tags')
+
 
 def getValueOverRange(word, startDate, num_day):
     to_return = 0

bin/RegexForTermsFrequency.py

@@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -129,6 +131,11 @@ if __name__ == "__main__":
                             for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
                                 sendEmailNotification(email, 'Term', mail_body)
 
+                        # tag paste
+                        for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
+                            msg = '{};{}'.format(tag, filename)
+                            p.populate_set_out(msg, 'Tags')
+
                         set_name = 'regex_' + dico_regexname_to_redis[regex_str]
                         new_to_the_set = server_term.sadd(set_name, filename)
                         new_to_the_set = True if new_to_the_set == 1 else False

bin/SetForTermsFrequency.py

@@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -121,6 +123,11 @@ if __name__ == "__main__":
                         for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
                             sendEmailNotification(email, 'Term', mail_body)
 
+                    # tag paste
+                    for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
+                        msg = '{};{}'.format(tag, filename)
+                        p.populate_set_out(msg, 'Tags')
+
                     print(the_set, "matched in", filename)
                     set_name = 'set_' + dico_setname_to_redis[the_set]
                     new_to_the_set = server_term.sadd(set_name, filename)

bin/packages/modules.cfg

@@ -32,13 +32,15 @@ publish = Redis_Words
 
 [Curve]
 subscribe = Redis_Words
-publish = Redis_CurveManageTopSets
+publish = Redis_CurveManageTopSets,Redis_Tags
 
 [RegexForTermsFrequency]
 subscribe = Redis_Global
+publish = Redis_Tags
 
 [SetForTermsFrequency]
 subscribe = Redis_Global
+publish = Redis_Tags
 
 [CurveManageTopSets]
 subscribe = Redis_CurveManageTopSets

var/www/modules/terms/Flask_terms.py

@@ -24,6 +24,7 @@ cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_term = Flask_config.r_serv_term
 r_serv_cred = Flask_config.r_serv_cred
+bootstrap_label = Flask_config.bootstrap_label
 
 terms = Blueprint('terms', __name__, template_folder='templates')
 
@@ -51,6 +52,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
 # same value as in `bin/NotificationHelper.py`
 # Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
 TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
 
 '''CRED'''
 REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@@ -152,6 +154,7 @@ def terms_management():
 
     # Maps a specific term to the associated email addresses
     notificationEMailTermMapping = {}
+    notificationTagsTermMapping = {}
 
     #Regex
     trackReg_list = []
@@ -160,6 +163,7 @@ def terms_management():
     for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
 
         notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) )
+        notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
 
         if tracked_regex not in notificationEnabledDict:
             notificationEnabledDict[tracked_regex] = False
@@ -186,7 +190,7 @@ def terms_management():
         tracked_set = tracked_set
 
         notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) )
-
+        notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
 
         if tracked_set not in notificationEnabledDict:
             notificationEnabledDict[tracked_set] = False
@@ -212,6 +216,7 @@ def terms_management():
     for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
 
         notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
+        notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
 
         if tracked_term not in notificationEnabledDict:
             notificationEnabledDict[tracked_term] = False
@@ -239,12 +244,14 @@ def terms_management():
         term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
         black_list.append([blacked_term, term_date])
 
+    print(notificationTagsTermMapping)
     return render_template("terms_management.html",
             black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
             track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
             trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
             trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
-            per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
+            per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
+            notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
 
 
 @terms.route("/terms_management_query_paste/")
@@ -313,6 +320,7 @@ def terms_management_action():
     action = request.args.get('action')
     term =  request.args.get('term')
     notificationEmailsParam = request.args.get('emailAddresses')
+    input_tags = request.args.get('tags')
 
     if action is None or term is None or notificationEmailsParam is None:
         return "None"
@@ -334,6 +342,8 @@ def terms_management_action():
                     if re.match(r"[^@]+@[^@]+\.[^@]+", email):
                         validNotificationEmails.append(email)
 
+                # create tags list
+                list_tags = input_tags.split()
 
                 # check if regex/set or simple term
                 #regex
@@ -345,6 +355,9 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
 
                 #set
                 elif term.startswith('\\') and term.endswith('\\'):
@@ -363,6 +376,9 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
 
                 #simple term
                 else:
@@ -373,6 +389,9 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
 
             elif action == "toggleEMailNotification":
                 # get the current state
@@ -397,6 +416,8 @@ def terms_management_action():
 
                 # delete the associated notification emails too
                 r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
+                # delete the associated tags set
+                r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
 
         elif section == "blacklistTerm":
             if action == "add":

var/www/modules/terms/templates/terms_management.html

@@ -98,7 +98,8 @@
                             <div class="form-group input-group" style="margin-bottom: 30px;">
                                 <span class="input-group-addon"><span class="fa fa-eye"></span></span>
                                 <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
-                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
+                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, comma separated)" type="text" style="max-width: 400px;">
+                                <input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
                                 <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
                             </div>
 
@@ -119,7 +120,16 @@
                                 <!-- SET -->
                                 {% for set in trackSet_list %}
                                   <tr style="background-color: #cdffca;">
-                                      <td>{{ set }}</td>
+                                      <td>
+                                          <span class="term_name">{{ set }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[set] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                          </div>
+                                      </td>
                                       <td>{{ trackSet_list_values[loop.index0][3] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][0] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][1] }}</td>
@@ -136,7 +146,16 @@
                                 <!-- REGEX -->
                                 {% for regex in trackReg_list %}
                                   <tr style="background-color: #fffdca;">
-                                      <td>{{ regex }}</td>
+                                      <td>
+                                          <span class="term_name">{{ regex }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[regex] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                          </div>
+                                      </td>
                                       <td>{{ trackReg_list_values[loop.index0][3] }}</td>
                                       <td>{{ trackReg_list_values[loop.index0][0] }}</td>
                                       <td>{{ trackReg_list_values[loop.index0][1] }}</td>
@@ -153,7 +172,16 @@
                                 <!-- Normal term -->
                                 {% for term in track_list %}
                                   <tr>
-                                      <td>{{ term }}</td>
+                                      <td>
+                                          <span class="term_name">{{ term }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[term] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                          </div>
+                                      </td>
                                       <td>{{ track_list_values[loop.index0][3] }}</td>
                                       <td>{{ track_list_values[loop.index0][0] }}</td>
                                       <td>{{ track_list_values[loop.index0][1] }}</td>
@@ -351,17 +379,19 @@ function perform_binding() {
 function perform_operation(){
     var curr_section = $(this).attr('data-section');
     var curr_action = $(this).attr('data-action');
+    var row_tr = $(this).closest("tr");
     if (curr_action == "add") {
         var curr_term = $('#'+curr_section+'Input').val();
         var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
+        var tags = $('#followTermTag').val();
     } else {
         var curr_term = $(this).attr('data-content');
         var email_addresses = "";
     }
-    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
+    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
 
     if (curr_term != "") {
-        console.log(data_to_send);
+        //console.log(data_to_send);
         $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
             if(status == "success") {
                 var json = data;
@@ -372,13 +402,8 @@ function perform_operation(){
                         $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
                             reload_per_paste();
                         });
-                   } else if (json.action == "delete") {
-                       // Find indexes of row which have the term in the first column
-                       var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
-                           console.log(table_track.cell( rowIdx, 0 ).data())
-                           return table_track.cell( rowIdx, 0 ).data() === json.term;
-                       } );
-                       table_track.rows(index).remove().draw( false );
+                    } else if (json.action == "delete") {
+                      row_tr.remove()
                     }
                 } else if(json.section == "blacklistTerm"){
                     if(json.action == "add") {