merge master

HOWTO.md

@@ -114,6 +114,9 @@ There are two types of installation. You can install a *local* or a *remote* Spl
 2. *(Splash host)* To install and setup your tor proxy:
     - Install the tor proxy: ``sudo apt-get install tor -y``
         (Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL)
+    - Allow Tor to bind to any interface or to the docker interface (by default binds to 127.0.0.1 only) in ``/etc/tor/torrc``
+        ``SOCKSPort 0.0.0.0:9050`` or
+        ``SOCKSPort 172.17.0.1:9050``
     - Add the following line ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc``
       (for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform)
     - Restart the tor proxy: ``sudo service tor restart``

OVERVIEW.md

@@ -26,6 +26,24 @@ ARDB overview
 ARDB_DB
 * DB 1 - Curve
 * DB 2 - TermFreq
+	----------------------------------------- TERM ----------------------------------------
+
+	SET - 'TrackedRegexSet'				term
+
+	HSET - 'TrackedRegexDate'			tracked_regex		today_timestamp
+
+	SET - 'TrackedSetSet'				set_to_add
+
+	HSET - 'TrackedSetDate'				set_to_add		today_timestamp
+
+	SET - 'TrackedSetTermSet'			term
+
+	HSET - 'TrackedTermDate'			tracked_regex		today_timestamp
+
+	SET - 'TrackedNotificationEmails_'+term/set	email
+
+	SET - 'TrackedNotifications'			term/set
+
 * DB 3 - Trending
 * DB 4 - Sentiment
 * DB 5 - TermCred

bin/Curve.py

@@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
             for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
                 sendEmailNotification(email, 'Term', mail_body)
 
+        # tag paste
+        for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
+            msg = '{};{}'.format(tag, path)
+            p.populate_set_out(msg, 'Tags')
+
 
 def getValueOverRange(word, startDate, num_day):
     to_return = 0

bin/MISP_The_Hive_feeder.py

@@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
 
 
 
-def create_the_hive_alert(source, path, content, tag):
+def create_the_hive_alert(source, path, tag):
     tags = list(r_serv_metadata.smembers('tag:'+path))
 
     artifacts = [
@@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag):
     ]
 
     l_tags = tag.split(',')
-    print(tag)
 
     # Prepare the sample Alert
     sourceRef = str(uuid.uuid4())[0:6]
@@ -91,6 +90,41 @@ def create_the_hive_alert(source, path, content, tag):
     except:
         print('hive connection error')
 
+def feeder(message, count=0):
+
+    if flag_the_hive or flag_misp:
+        tag, path = message.split(';')
+        ## FIXME: remove it
+        if PASTES_FOLDER not in path:
+            path = os.path.join(PASTES_FOLDER, path)
+        try:
+            paste = Paste.Paste(path)
+        except FileNotFoundError:
+            if count < 10:
+                r_serv_db.zincrby('mess_not_saved_export', message, 1)
+                return 0
+            else:
+                r_serv_db.zrem('mess_not_saved_export', message)
+                print('Error: {} do not exist, tag= {}'.format(path, tag))
+                return 0
+
+        source = '/'.join(paste.p_path.split('/')[-6:])
+
+        if HiveApi != False:
+            if int(r_serv_db.get('hive:auto-alerts')) == 1:
+                whitelist_hive = r_serv_db.scard('whitelist_hive')
+                if r_serv_db.sismember('whitelist_hive', tag):
+                    create_the_hive_alert(source, path, tag)
+            else:
+                print('hive, auto alerts creation disable')
+        if flag_misp:
+            if int(r_serv_db.get('misp:auto-events')) == 1:
+                if r_serv_db.sismember('whitelist_misp', tag):
+                    misp_wrapper.pushToMISP(uuid_ail, path, tag)
+            else:
+                print('misp, auto events creation disable')
+
+
 if __name__ == "__main__":
 
     publisher.port = 6380
@@ -119,10 +153,18 @@ if __name__ == "__main__":
         db=cfg.getint("ARDB_Metadata", "db"),
         decode_responses=True)
 
+    # set sensor uuid
     uuid_ail = r_serv_db.get('ail:uuid')
     if uuid_ail is None:
         uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
 
+    # set default
+    if r_serv_db.get('hive:auto-alerts') is None:
+        r_serv_db.set('hive:auto-alerts', 0)
+
+    if r_serv_db.get('misp:auto-events') is None:
+        r_serv_db.set('misp:auto-events', 0)
+
     p = Process(config_section)
     # create MISP connection
     if flag_misp:
@@ -167,36 +209,30 @@ if __name__ == "__main__":
             r_serv_db.set('ail:thehive', False)
             print('Not connected to The HIVE')
 
+    refresh_time = 3
+    ## FIXME: remove it
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+    time_1 = time.time()
+
     while True:
 
         # Get one message from the input queue
         message = p.get_from_set()
         if message is None:
+
+            # handle not saved pastes
+            if int(time.time() - time_1) > refresh_time:
+
+                num_queu = r_serv_db.zcard('mess_not_saved_export')
+                list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1,  withscores=True)
+
+                if num_queu and list_queu:
+                    for i in range(0, num_queu):
+                        feeder(list_queu[i][0],list_queu[i][1])
+
+                time_1 = time.time()
+            else:
                 publisher.debug("{} queue is empty, waiting 1s".format(config_section))
                 time.sleep(1)
-            continue
         else:
-
+            feeder(message)
-            if flag_the_hive or flag_misp:
-                tag, path = message.split(';')
-                paste = Paste.Paste(path)
-                source = '/'.join(paste.p_rel_path.split('/')[-6:])
-
-                full_path = os.path.join(os.environ['AIL_HOME'],
-                                        p.config.get("Directories", "pastes"), path)
-
-
-                if HiveApi != False:
-                    if int(r_serv_db.get('hive:auto-alerts')) == 1:
-                        whitelist_hive = r_serv_db.scard('whitelist_hive')
-                        if r_serv_db.sismember('whitelist_hive', tag):
-                            create_the_hive_alert(source, path, full_path, tag)
-
-                    else:
-                        print('hive, auto alerts creation disable')
-                if flag_misp:
-                    if int(r_serv_db.get('misp:auto-events')) == 1:
-                        if r_serv_db.sismember('whitelist_misp', tag):
-                            misp_wrapper.pushToMISP(uuid_ail, path, tag)
-                    else:
-                        print('misp, auto events creation disable')

bin/Mixer.py

@@ -80,6 +80,7 @@ if __name__ == '__main__':
     # OTHER CONFIG #
     operation_mode = cfg.getint("Module_Mixer", "operation_mode")
     ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
+    default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
 
     PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
 
@@ -107,10 +108,11 @@ if __name__ == '__main__':
                         feeder_name = feeder_name.split('/')[1]
 
                 except ValueError as e:
-                    feeder_name = "unnamed_feeder"
+                    feeder_name = default_unnamed_feed_name
+                    paste_name = complete_paste
 
                 # remove absolute path
-                paste_name = complete_paste.replace(PASTES_FOLDER, '', 1)
+                paste_name = paste_name.replace(PASTES_FOLDER, '', 1)
 
                 # Processed paste
                 processed_paste += 1

bin/NotificationHelper.py

@@ -3,8 +3,10 @@
 
 import argparse
 import configparser
+import traceback
 import os
 import smtplib
+from pubsublogger import publisher
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
@@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti
 # CONFIG #
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
 
+publisher.port = 6380
+publisher.channel = "Script"
+
 # notifications enabled/disabled
 TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
 
@@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
 # Keys will be e.g. TrackedNotificationEmails<TERMNAME>
 TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
 
-
 def sendEmailNotification(recipient, alert_name, content):
 
     if not os.path.exists(configfile):
@@ -33,22 +37,12 @@ def sendEmailNotification(recipient, alert_name, content):
     cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
-    sender = cfg.get("Notifications", "sender"),
+    sender = cfg.get("Notifications", "sender")
-    sender_host = cfg.get("Notifications", "sender_host"),
+    sender_host = cfg.get("Notifications", "sender_host")
-    sender_port = cfg.getint("Notifications", "sender_port"),
+    sender_port = cfg.getint("Notifications", "sender_port")
-    sender_pw = cfg.get("Notifications", "sender_pw"),
+    sender_pw = cfg.get("Notifications", "sender_pw")
-
+    if sender_pw == 'None':
-    if isinstance(sender, tuple):
+        sender_pw = None
-        sender = sender[0]
-
-    if isinstance(sender_host, tuple):
-        sender_host = sender_host[0]
-
-    if isinstance(sender_port, tuple):
-        sender_port = sender_port[0]
-
-    if isinstance(sender_pw, tuple):
-        sender_pw = sender_pw[0]
 
     # raise an exception if any of these is None
     if (sender is None or
@@ -83,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content):
         smtp_server.quit()
         print('Send notification ' + alert_name + ' to '+recipient)
 
-    except Exception as e:
+    except Exception as err:
-        print(str(e))
+        traceback.print_tb(err.__traceback__)
-        # raise e
+        publisher.warning(err)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Test notification sender.')

bin/RegexForTermsFrequency.py

@@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -129,6 +131,11 @@ if __name__ == "__main__":
                             for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
                                 sendEmailNotification(email, 'Term', mail_body)
 
+                        # tag paste
+                        for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
+                            msg = '{};{}'.format(tag, filename)
+                            p.populate_set_out(msg, 'Tags')
+
                         set_name = 'regex_' + dico_regexname_to_redis[regex_str]
                         new_to_the_set = server_term.sadd(set_name, filename)
                         new_to_the_set = True if new_to_the_set == 1 else False

bin/SetForTermsFrequency.py

@@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
 
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
 # create direct link in mail
 full_paste_url = "/showsavedpaste/?paste="
 
@@ -121,6 +123,11 @@ if __name__ == "__main__":
                         for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
                             sendEmailNotification(email, 'Term', mail_body)
 
+                    # tag paste
+                    for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
+                        msg = '{};{}'.format(tag, filename)
+                        p.populate_set_out(msg, 'Tags')
+
                     print(the_set, "matched in", filename)
                     set_name = 'set_' + dico_setname_to_redis[the_set]
                     new_to_the_set = server_term.sadd(set_name, filename)

bin/packages/config.cfg.sample

@@ -27,6 +27,7 @@ ail_domain = http://localhost:7000
 sender = sender@example.com
 sender_host = smtp.example.com
 sender_port = 1337
+sender_pw = None
 
 # optional for using with authenticated SMTP over SSL
 # sender_pw = securepassword
@@ -98,6 +99,7 @@ threshold_stucked_module=600
 operation_mode = 3
 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
 ttl_duplicate = 86400
+default_unnamed_feed_name = unnamed_feeder
 
 [RegexForTermsFrequency]
 max_execution_time = 60

bin/packages/modules.cfg

@@ -32,13 +32,15 @@ publish = Redis_Words
 
 [Curve]
 subscribe = Redis_Words
-publish = Redis_CurveManageTopSets
+publish = Redis_CurveManageTopSets,Redis_Tags
 
 [RegexForTermsFrequency]
 subscribe = Redis_Global
+publish = Redis_Tags
 
 [SetForTermsFrequency]
 subscribe = Redis_Global
+publish = Redis_Tags
 
 [CurveManageTopSets]
 subscribe = Redis_CurveManageTopSets

var/www/modules/Flask_config.py

@@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis(
     db=cfg.getint("ARDB_Onion", "db"),
     decode_responses=True)
 
-
 sys.path.append('../../configs/keys')
 # MISP #
 try:

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -506,6 +506,8 @@ def edit_tag_export():
     status_misp = []
     status_hive = []
 
+    infoleak_tags = Taxonomies().get('infoleak').machinetags()
+    is_infoleak_tag = []
 
     for tag in list_export_tags:
         if r_serv_db.sismember('whitelist_misp', tag):
@@ -519,6 +521,11 @@ def edit_tag_export():
         else:
             status_hive.append(False)
 
+        if tag in infoleak_tags:
+            is_infoleak_tag.append(True)
+        else:
+            is_infoleak_tag.append(False)
+
     if misp_auto_events is not None:
         if int(misp_auto_events) == 1:
             misp_active = True
@@ -543,6 +550,7 @@ def edit_tag_export():
                             misp_active=misp_active,
                             hive_active=hive_active,
                             list_export_tags=list_export_tags,
+                            is_infoleak_tag=is_infoleak_tag,
                             status_misp=status_misp,
                             status_hive=status_hive,
                             nb_tags_whitelist_misp=nb_tags_whitelist_misp,
@@ -594,5 +602,37 @@ def disable_hive_auto_alert():
     r_serv_db.set('hive:auto-alerts', 0)
     return edit_tag_export()
 
+@PasteSubmit.route("/PasteSubmit/add_push_tag")
+def add_push_tag():
+    tag = request.args.get('tag')
+    if tag is not None:
+
+        #limit tag length
+        if len(tag) > 49:
+            tag = tag[0:48]
+
+        r_serv_db.sadd('list_export_tags', tag)
+
+        to_return = {}
+        to_return["tag"] = tag
+        return jsonify(to_return)
+    else:
+        return 'None args', 400
+
+@PasteSubmit.route("/PasteSubmit/delete_push_tag")
+def delete_push_tag():
+    tag = request.args.get('tag')
+
+    infoleak_tags = Taxonomies().get('infoleak').machinetags()
+    if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag):
+        r_serv_db.srem('list_export_tags', tag)
+        r_serv_db.srem('whitelist_misp', tag)
+        r_serv_db.srem('whitelist_hive', tag)
+        to_return = {}
+        to_return["tag"] = tag
+        return jsonify(to_return)
+    else:
+        return 'this tag can\'t be removed', 400
+
 # ========= REGISTRATION =========
 app.register_blueprint(PasteSubmit, url_prefix=baseUrl)

var/www/modules/PasteSubmit/templates/edit_tag_export.html

@@ -37,6 +37,9 @@
 				background: #d91f2d;
 				color: #fff;
 			}
+			.mouse_pointer{
+				cursor: pointer;
+			}
 		</style>
 
 	</head>
@@ -169,7 +172,14 @@
 																<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
 															{% endif %}
 						                </td>
-														<td>{{ tag }}</td>
+														<td>
+															{{ tag }}
+															{% if not is_infoleak_tag[loop.index0] %}
+																<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
+																	<span class="glyphicon glyphicon-trash"></span>
+																</div>
+															{% endif %}
+														</td>
 												</tr>
 											{%  endfor %}
 
@@ -209,7 +219,14 @@
 																<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
 															{% endif %}
 						                </td>
-														<td>{{ tag }}</td>
+														<td>
+															{{ tag }}
+															{% if not is_infoleak_tag[loop.index0] %}
+																<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
+																	<span class="glyphicon glyphicon-trash"></span>
+																</div>
+															{% endif %}
+														</td>
 												</tr>
 											{%  endfor %}
 
@@ -232,6 +249,42 @@
 
       </form>
 
+			<div>
+
+				<div id="add_custom_tag_modal" class="modal fade" role="dialog">
+					<div class="modal-dialog">
+
+						<!-- Modal content-->
+						<div id="mymodalcontent" class="modal-content">
+							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+								<h2 class="text-center">Add Custom Tag</h2>
+
+							</div>
+
+							<div class="modal-body">
+								<div class="form-group input-group" style="margin-bottom: 0px;">
+                  <span class="input-group-addon"><i class="fa fa-tag fa"></i></span>
+                  <input id="new_custom_tag" class="form-control" placeholder="Add a new custom tag to the tag export" type="text">
+                </div>
+							</div>
+
+							<div class="modal-footer">
+								<button class="btn btn-primary btn-tags" onclick="add_custom_tag()">
+										<span class="glyphicon glyphicon-plus"></span>
+										<span class="label-icon">Add Custom Tag</span>
+								</button>
+								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+							</div>
+						</div>
+					</div>
+				</div>
+
+				<button type="button" class="btn btn-primary pull-right" data-toggle="modal" data-target="#add_custom_tag_modal" data-url="{{ url_for('Tags.taxonomies') }}">
+					<span class="glyphicon glyphicon-plus "></span> Add Custom Tag
+				</button>
+			</div>
+
       </div>
             <!-- /#page-wrapper -->
 
@@ -277,6 +330,25 @@ $(document).ready(function(){
 			}
 		);
   }
+
+	function delete_push_tag(tag){
+		//var row_tr = $(this).closest("tr");
+		$.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){
+				if(status == "success") {
+						//row_tr.remove();
+						window.location.reload(false);
+				}
+		});
+	}
+
+	function add_custom_tag(){
+		$.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){
+				if(status == "success") {
+						//row_tr.remove();
+						window.location.reload(false);
+				}
+		});
+	}
 </script>
 
 </html>

var/www/modules/search/Flask_search.py

@@ -16,6 +16,8 @@ from whoosh import index
 from whoosh.fields import Schema, TEXT, ID
 from whoosh.qparser import QueryParser
 
+import time
+
 # ============ VARIABLES ============
 import Flask_config
 
@@ -55,8 +57,8 @@ def get_index_list(selected_index=""):
         if os.path.isdir(os.path.join(baseindexpath, dirs)):
             value = dirs
             name = to_iso_date(dirs) + " - " + \
-                    str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \
+                    str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \
-                    "(" + str(get_item_count(dirs)) + " Items" + ")"
+                    #"(" + str(get_item_count(dirs))''' + " Items" + ")"
             flag = dirs==selected_index.split('/')[-1]
             if dirs == "old_index":
                 temp = [value, name, flag]
@@ -66,6 +68,7 @@ def get_index_list(selected_index=""):
     index_list.sort(reverse=True, key=lambda x: x[0])
     if len(temp) != 0:
         index_list.append(temp)
+
     return index_list
 
 def get_dir_size(directory):
@@ -108,6 +111,7 @@ def search():
     else:
         selected_index = os.path.join(baseindexpath, index_name)
 
+    ''' temporary disabled
     # Search filename
     for path in r_serv_pasteName.smembers(q[0]):
         r.append(path)
@@ -119,6 +123,7 @@ def search():
         curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
         paste_date.append(curr_date)
         paste_size.append(paste._get_p_size())
+    '''
 
     # Search full line
     schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
@@ -159,15 +164,18 @@ def search():
         results = searcher.search(query)
         num_res = len(results)
 
+    index_list = get_index_list()
+
     index_min = 1
-    index_max = len(get_index_list())
+    index_max = len(index_list)
+
     return render_template("search.html", r=r, c=c,
             query=request.form['query'], paste_date=paste_date,
             paste_size=paste_size, char_to_display=max_preview_modal,
             num_res=num_res, index_min=index_min, index_max=index_max,
             bootstrap_label=bootstrap_label,
             paste_tags=paste_tags,
-            index_list=get_index_list(selected_index)
+            index_list=index_list
            )
 
 
@@ -212,6 +220,7 @@ def get_more_search_result():
             p_tags = r_serv_metadata.smembers('tag:'+path)
             l_tags = []
             for tag in p_tags:
+                complete_tag = tag
                 tag = tag.split('=')
                 if len(tag) > 1:
                     if tag[1] != '':
@@ -223,7 +232,7 @@ def get_more_search_result():
                 else:
                     tag = tag[0]
 
-                l_tags.append(tag)
+                l_tags.append( (tag, complete_tag) )
             list_tags.append(l_tags)
 
         to_return = {}

var/www/modules/search/templates/search.html

@@ -201,7 +201,9 @@
                 var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
                 var tag = ""
                 for(j=0; j<data.list_tags[i].length; j++) {
-                  tag = tag + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[j] + "</span>"
+                  tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
+                        + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
+                        + "</span>" + "</a>"
                 }
                 search_table.row.add( [
                     init_num_of_elements_in_table+((offset))+i+1,

var/www/modules/terms/Flask_terms.py

@@ -10,7 +10,7 @@ import redis
 import datetime
 import calendar
 import flask
-from flask import Flask, render_template, jsonify, request, Blueprint
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
 import re
 import Paste
 from pprint import pprint
@@ -24,6 +24,8 @@ cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_term = Flask_config.r_serv_term
 r_serv_cred = Flask_config.r_serv_cred
+r_serv_db = Flask_config.r_serv_db
+bootstrap_label = Flask_config.bootstrap_label
 
 terms = Blueprint('terms', __name__, template_folder='templates')
 
@@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
 # same value as in `bin/NotificationHelper.py`
 # Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
 TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
 
 '''CRED'''
 REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False):
             filtered_usernames.append(usr)
     return filtered_usernames
 
+def save_tag_to_auto_push(list_tag):
+    for tag in set(list_tag):
+        #limit tag length
+        if len(tag) > 49:
+            tag = tag[0:48]
+        r_serv_db.sadd('list_export_tags', tag)
 
 # ============ ROUTES ============
 
@@ -152,6 +161,7 @@ def terms_management():
 
     # Maps a specific term to the associated email addresses
     notificationEMailTermMapping = {}
+    notificationTagsTermMapping = {}
 
     #Regex
     trackReg_list = []
@@ -159,7 +169,8 @@ def terms_management():
     trackReg_list_num_of_paste = []
     for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
 
-        notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) )
+        notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
+        notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
 
         if tracked_regex not in notificationEnabledDict:
             notificationEnabledDict[tracked_regex] = False
@@ -185,8 +196,8 @@ def terms_management():
     for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
         tracked_set = tracked_set
 
-        notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) )
+        notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
-
+        notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
 
         if tracked_set not in notificationEnabledDict:
             notificationEnabledDict[tracked_set] = False
@@ -211,7 +222,8 @@ def terms_management():
     track_list_num_of_paste = []
     for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
 
-        notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
+        notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
+        notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
 
         if tracked_term not in notificationEnabledDict:
             notificationEnabledDict[tracked_term] = False
@@ -244,7 +256,8 @@ def terms_management():
             track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
             trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
             trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
-            per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
+            per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
+            notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
 
 
 @terms.route("/terms_management_query_paste/")
@@ -313,6 +326,7 @@ def terms_management_action():
     action = request.args.get('action')
     term =  request.args.get('term')
     notificationEmailsParam = request.args.get('emailAddresses')
+    input_tags = request.args.get('tags')
 
     if action is None or term is None or notificationEmailsParam is None:
         return "None"
@@ -320,11 +334,8 @@ def terms_management_action():
         if section == "followTerm":
             if action == "add":
 
-                # Strip all whitespace
-                notificationEmailsParam = "".join(notificationEmailsParam.split())
-
                 # Make a list of all passed email addresses
-                notificationEmails = notificationEmailsParam.split(",")
+                notificationEmails = notificationEmailsParam.split()
 
                 validNotificationEmails = []
                 # check for valid email addresses
@@ -334,6 +345,8 @@ def terms_management_action():
                     if re.match(r"[^@]+@[^@]+\.[^@]+", email):
                         validNotificationEmails.append(email)
 
+                # create tags list
+                list_tags = input_tags.split()
 
                 # check if regex/set or simple term
                 #regex
@@ -345,6 +358,10 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
+                    save_tag_to_auto_push(list_tags)
 
                 #set
                 elif term.startswith('\\') and term.endswith('\\'):
@@ -363,6 +380,10 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
+                    save_tag_to_auto_push(list_tags)
 
                 #simple term
                 else:
@@ -373,6 +394,10 @@ def terms_management_action():
                         r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
                     # enable notifications by default
                     r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
+                    # add tags list
+                    for tag in list_tags:
+                        r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
+                    save_tag_to_auto_push(list_tags)
 
             elif action == "toggleEMailNotification":
                 # get the current state
@@ -397,6 +422,8 @@ def terms_management_action():
 
                 # delete the associated notification emails too
                 r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
+                # delete the associated tags set
+                r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
 
         elif section == "blacklistTerm":
             if action == "add":
@@ -413,6 +440,28 @@ def terms_management_action():
         to_return["term"] = term
         return jsonify(to_return)
 
+@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
+def delete_terms_tags():
+    term = request.form.get('term')
+    tags_to_delete = request.form.getlist('tags_to_delete')
+
+    if term is not None and tags_to_delete is not None:
+        for tag in tags_to_delete:
+            r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
+        return redirect(url_for('terms.terms_management'))
+    else:
+        return 'None args', 400
+
+@terms.route("/terms_management/delete_terms_email", methods=['GET'])
+def delete_terms_email():
+    term =  request.args.get('term')
+    email =  request.args.get('email')
+
+    if term is not None and email is not None:
+        r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
+        return redirect(url_for('terms.terms_management'))
+    else:
+        return 'None args', 400
 
 
 @terms.route("/terms_plot_tool/")

var/www/modules/terms/templates/terms_management.html

@@ -36,6 +36,12 @@
          white-space:pre-wrap;
          word-wrap:break-word;
       }
+      .mouse_pointer{
+				cursor: pointer;
+			}
+      .lb-md {
+        font-size: 16px;
+      }
   </style>
 </head>
 <body>
@@ -74,6 +80,7 @@
     <div class="row">
         <div class="col-lg-12">
             <div class="row">
+              {% set uniq_id = namespace(modal_id=0)%}
                 <div class="col-lg-12">
                    <label class="switch">
                        <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
@@ -98,7 +105,8 @@
                             <div class="form-group input-group" style="margin-bottom: 30px;">
                                 <span class="input-group-addon"><span class="fa fa-eye"></span></span>
                                 <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
-                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
+                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
+                                <input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
                                 <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
                             </div>
 
@@ -119,7 +127,57 @@
                                 <!-- SET -->
                                 {% for set in trackSet_list %}
                                   <tr style="background-color: #cdffca;">
-                                      <td>{{ set }}</td>
+                                      <td>
+                                          <span class="term_name">{{ set }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[set] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                            {% if notificationTagsTermMapping[set] %}
+                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+                                      					<div class="modal-dialog">
+
+                                      						<!-- Modal content-->
+                                      						<div id="mymodalcontent" class="modal-content">
+                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+                                      								<h2 class="text-center">Remove Custom Tag</h2>
+
+                                      							</div>
+
+                                      							<div class="modal-body">
+                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+                                                        {% for tag in notificationTagsTermMapping[set] %}
+                                                          <div class="form-check">
+                                                             <input type="hidden" class="form-control" name="term" value="{{ set }}">
+                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+                                                            <label class="form-check-label">
+                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+                                                            </label>
+                                                            <br>
+                                                          </div>
+                                                        {%  endfor %}
+                                                      </form>
+                                      							</div>
+
+                                      							<div class="modal-footer">
+                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+                                      										<span class="glyphicon glyphicon-trash"></span>
+                                      										<span class="label-icon">Remove Tags</span>
+                                      								</button>
+                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+                                      							</div>
+                                      						</div>
+                                      					</div>
+                                      				</div>
+                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+                                            {%  endif %}
+                                          </div>
+                                      </td>
                                       <td>{{ trackSet_list_values[loop.index0][3] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][0] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][1] }}</td>
@@ -130,13 +188,73 @@
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
                                           &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
                                       </p></td>
-                                      <td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td>
+                                      <td>
+                                        {% for email in notificationEMailTermMapping[set] %}
+                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
+                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+                                                <span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
+                                            </div>
+                                          </a>
+                                          {{ email }}
+                                          <br>
+                                        {%  endfor %}
+                                      </td>
                                   </tr>
                                 {% endfor %}
                                 <!-- REGEX -->
                                 {% for regex in trackReg_list %}
                                   <tr style="background-color: #fffdca;">
-                                      <td>{{ regex }}</td>
+                                      <td>
+                                          <span class="term_name">{{ regex }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[regex] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                            {% if notificationTagsTermMapping[regex] %}
+                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+                                      					<div class="modal-dialog">
+
+                                      						<!-- Modal content-->
+                                      						<div id="mymodalcontent" class="modal-content">
+                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+                                      								<h2 class="text-center">Remove Custom Tag</h2>
+
+                                      							</div>
+
+                                      							<div class="modal-body">
+                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+                                                        {% for tag in notificationTagsTermMapping[regex] %}
+                                                          <div class="form-check">
+                                                             <input type="hidden" class="form-control" name="term" value="{{ regex }}">
+                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+                                                            <label class="form-check-label">
+                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+                                                            </label>
+                                                            <br>
+                                                          </div>
+                                                        {%  endfor %}
+                                                      </form>
+                                      							</div>
+
+                                      							<div class="modal-footer">
+                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+                                      										<span class="glyphicon glyphicon-trash"></span>
+                                      										<span class="label-icon">Remove Tags</span>
+                                      								</button>
+                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+                                      							</div>
+                                      						</div>
+                                      					</div>
+                                      				</div>
+                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+                                            {%  endif %}
+                                          </div>
+                                      </td>
                                       <td>{{ trackReg_list_values[loop.index0][3] }}</td>
                                       <td>{{ trackReg_list_values[loop.index0][0] }}</td>
                                       <td>{{ trackReg_list_values[loop.index0][1] }}</td>
@@ -147,13 +265,73 @@
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
                                           &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
                                       </p></td>
-                                      <td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td>
+                                      <td>
+                                        {% for email in notificationEMailTermMapping[regex] %}
+                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
+                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+                                              <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
+                                            </div>
+                                          </a>
+                                          {{ email }}
+                                          <br>
+                                        {%  endfor %}
+                                      </td>
                                   </tr>
                                 {% endfor %}
                                 <!-- Normal term -->
                                 {% for term in track_list %}
                                   <tr>
-                                      <td>{{ term }}</td>
+                                      <td>
+                                          <span class="term_name">{{ term }}</span>
+                                          <div>
+                                            {% for tag in notificationTagsTermMapping[term] %}
+                                            <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+                                            </a>
+                                            {%  endfor %}
+                                            {% if notificationTagsTermMapping[term] %}
+                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+                                      					<div class="modal-dialog">
+
+                                      						<!-- Modal content-->
+                                      						<div id="mymodalcontent" class="modal-content">
+                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+                                      								<h2 class="text-center">Remove Custom Tag</h2>
+
+                                      							</div>
+
+                                      							<div class="modal-body">
+                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+                                                        {% for tag in notificationTagsTermMapping[term] %}
+                                                          <div class="form-check">
+                                                             <input type="hidden" class="form-control" name="term" value="{{ term }}">
+                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+                                                            <label class="form-check-label">
+                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+                                                            </label>
+                                                            <br>
+                                                          </div>
+                                                        {%  endfor %}
+                                                      </form>
+                                      							</div>
+
+                                      							<div class="modal-footer">
+                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+                                      										<span class="glyphicon glyphicon-trash"></span>
+                                      										<span class="label-icon">Remove Tags</span>
+                                      								</button>
+                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+                                      							</div>
+                                      						</div>
+                                      					</div>
+                                      				</div>
+                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+                                            {%  endif %}
+                                          </div>
+                                      </td>
                                       <td>{{ track_list_values[loop.index0][3] }}</td>
                                       <td>{{ track_list_values[loop.index0][0] }}</td>
                                       <td>{{ track_list_values[loop.index0][1] }}</td>
@@ -164,7 +342,17 @@
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
                                           &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
                                       </p></td>
-                                      <td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
+                                      <td>
+                                        {% for email in notificationEMailTermMapping[term] %}
+                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
+                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+                                                <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
+                                            </div>
+                                          </a>
+                                          {{ email }}
+                                          <br>
+                                        {%  endfor %}
+                                      </td>
                                   </tr>
                                 {% endfor %}
                                 </tbody>
@@ -351,17 +539,19 @@ function perform_binding() {
 function perform_operation(){
     var curr_section = $(this).attr('data-section');
     var curr_action = $(this).attr('data-action');
+    var row_tr = $(this).closest("tr");
     if (curr_action == "add") {
         var curr_term = $('#'+curr_section+'Input').val();
         var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
+        var tags = $('#followTermTag').val();
     } else {
         var curr_term = $(this).attr('data-content');
         var email_addresses = "";
     }
-    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
+    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
 
     if (curr_term != "") {
-        console.log(data_to_send);
+        //console.log(data_to_send);
         $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
             if(status == "success") {
                 var json = data;
@@ -373,12 +563,7 @@ function perform_operation(){
                             reload_per_paste();
                         });
                     } else if (json.action == "delete") {
-                       // Find indexes of row which have the term in the first column
+                      row_tr.remove()
-                       var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
-                           console.log(table_track.cell( rowIdx, 0 ).data())
-                           return table_track.cell( rowIdx, 0 ).data() === json.term;
-                       } );
-                       table_track.rows(index).remove().draw( false );
                     }
                 } else if(json.section == "blacklistTerm"){
                     if(json.action == "add") {

var/www/update_thirdparty.sh

@@ -86,11 +86,11 @@ popd
 #active virtualenv
 source ./../../AILENV/bin/activate
 #Update MISP Taxonomies and Galaxies
-python3 -m pip install git+https://github.com/MISP/PyTaxonomies
+python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade
-python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies
+python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade
 
 #Update PyMISP
-python3 -m pip install git+https://github.com/MISP/PyMISP
+python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade
 
 #Update the Hive
-python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py
+python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py --upgrade