From 44491e837e2dd8cb3d8b2cbde290897adbcdab71 Mon Sep 17 00:00:00 2001 From: Philipp Schmied Date: Tue, 27 Feb 2018 14:44:22 +0100 Subject: [PATCH 1/9] README.md: Add note --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 18e1ee09..81dfbb15 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -[![Build Status](https://travis-ci.org/CIRCL/AIL-framework.svg?branch=master)](https://travis-ci.org/CIRCL/AIL-framework) - AIL === ![Logo](./doc/logo/logo-small.png?raw=true "AIL logo") +*This is a development fork. The original repository can be found [here](https://github.com/CIRCL/AIL-framework)* + AIL framework - Framework for Analysis of Information Leaks AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information. From 5b1f0b021220915d69a76fa3e803dcb60d649ef8 Mon Sep 17 00:00:00 2001 From: Philipp Schmied Date: Tue, 27 Feb 2018 15:12:02 +0100 Subject: [PATCH 2/9] Implemented email notifications (bin: config.cfg additions, email sending via analyzer scripts; var: Changes to add notifications via terms management); terms_management: Fixed click handlers not being added to all tracked terms. --- RegexForTermsFrequency.py | 123 +++++++++++++++ SetForTermsFrequency.py | 134 ++++++++++++++++ bin/Curve.py | 11 +- bin/NotificationHelper.py | 75 +++++++++ bin/RegexForTermsFrequency.py | 16 +- bin/SetForTermsFrequency.py | 14 +- bin/packages/config.cfg.sample | 33 ++-- var/www/modules/terms/Flask_terms.py | 81 +++++++++- .../terms/templates/terms_management.html | 145 ++++++++++-------- 9 files changed, 542 insertions(+), 90 deletions(-) create mode 100755 RegexForTermsFrequency.py create mode 100755 SetForTermsFrequency.py create mode 100755 bin/NotificationHelper.py diff --git a/RegexForTermsFrequency.py b/RegexForTermsFrequency.py new file mode 100755 index 00000000..b5570ea9 --- /dev/null +++ b/RegexForTermsFrequency.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" +This Module is used for term frequency. +It processes every paste coming from the global module and test the regexs +supplied in the term webpage. + +""" +import redis +import time +from pubsublogger import publisher +from packages import lib_words +from packages import Paste +import os +from os import environ +import datetime +import calendar +import re +from Helper import Process + +# Email notifications +from NotificationHelper import * + +# Config Variables +DICO_REFRESH_TIME = 60 #s + +BlackListTermsSet_Name = "BlackListSetTermSet" +TrackedTermsSet_Name = "TrackedSetTermSet" +TrackedRegexSet_Name = "TrackedRegexSet" + +top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set +oneDay = 60*60*24 +top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] +top_termFreq_setName_week = ["TopTermFreq_set_week", 7] +top_termFreq_setName_month = ["TopTermFreq_set_month", 31] +top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] + + +def refresh_dicos(): + dico_regex = {} + dico_regexname_to_redis = {} + for regex_str in server_term.smembers(TrackedRegexSet_Name): + dico_regex[regex_str[1:-1]] = re.compile(regex_str[1:-1]) + dico_regexname_to_redis[regex_str[1:-1]] = regex_str + + return dico_regex, dico_regexname_to_redis + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'RegexForTermsFrequency' + p = Process(config_section) + + # REDIS # + server_term = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_TermFreq", "host"), + port=p.config.get("Redis_Level_DB_TermFreq", "port"), + db=p.config.get("Redis_Level_DB_TermFreq", "db")) + + # FUNCTIONS # + publisher.info("RegexForTermsFrequency script started") + + #compile the regex + dico_refresh_cooldown = time.time() + dico_regex, dico_regexname_to_redis = refresh_dicos() + + message = p.get_from_set() + + # Regex Frequency + while True: + + if message is not None: + if time.time() - dico_refresh_cooldown > DICO_REFRESH_TIME: + dico_refresh_cooldown = time.time() + dico_regex, dico_regexname_to_redis = refresh_dicos() + print('dico got refreshed') + + filename = message + temp = filename.split('/') + timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) + + curr_set = top_termFreq_setName_day[0] + str(timestamp) + content = Paste.Paste(filename).get_p_content() + + #iterate the word with the regex + for regex_str, compiled_regex in dico_regex.items(): + matched = compiled_regex.search(content) + + if matched is not None: #there is a match + print('regex matched {}'.format(regex_str)) + matched = matched.group(0) + + # Add in Regex track set only if term is not in the blacklist + if matched not in server_term.smembers(BlackListTermsSet_Name): + + # Send a notification only when the member is in the set + if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name): + + # Send to every associated email adress + for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched): + sendEmailNotification(email, matched) + + + set_name = 'regex_' + dico_regexname_to_redis[regex_str] + new_to_the_set = server_term.sadd(set_name, filename) + new_to_the_set = True if new_to_the_set == 1 else False + + #consider the num of occurence of this term + regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1))) + #1 term per paste + if new_to_the_set: + regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1))) + server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1)) + server_term.zincrby(curr_set, dico_regexname_to_redis[regex_str], float(1)) + else: + pass + + else: + publisher.debug("Script RegexForTermsFrequency is Idling") + print "sleeping" + time.sleep(5) + message = p.get_from_set() diff --git a/SetForTermsFrequency.py b/SetForTermsFrequency.py new file mode 100755 index 00000000..d6e9bef9 --- /dev/null +++ b/SetForTermsFrequency.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" +This Module is used for term frequency. +It processes every paste coming from the global module and test the sets +supplied in the term webpage. + +""" +import redis +import time +from pubsublogger import publisher +from packages import lib_words +from packages import Paste +import os +import datetime +import calendar +import re +import ast +from Helper import Process + +# Email notifications +from NotificationHelper import * + +# Config Variables +BlackListTermsSet_Name = "BlackListSetTermSet" +TrackedTermsSet_Name = "TrackedSetTermSet" +TrackedRegexSet_Name = "TrackedRegexSet" +TrackedSetSet_Name = "TrackedSetSet" + +top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set +oneDay = 60*60*24 +top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] +top_termFreq_setName_week = ["TopTermFreq_set_week", 7] +top_termFreq_setName_month = ["TopTermFreq_set_month", 31] +top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] + +def add_quote_inside_tab(tab): + quoted_tab = "[" + for elem in tab[1:-1].split(','): + elem = elem.lstrip().strip() + quoted_tab += "\'{}\', ".format(elem) + quoted_tab = quoted_tab[:-2] #remove trailing , + quoted_tab += "]" + return str(quoted_tab) + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'SetForTermsFrequency' + p = Process(config_section) + + # REDIS # + server_term = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_TermFreq", "host"), + port=p.config.get("Redis_Level_DB_TermFreq", "port"), + db=p.config.get("Redis_Level_DB_TermFreq", "db")) + + # FUNCTIONS # + publisher.info("RegexForTermsFrequency script started") + + #get the dico and matching percent + dico_percent = {} + dico_set_tab = {} + dico_setname_to_redis = {} + for set_str in server_term.smembers(TrackedSetSet_Name): + tab_set = set_str[1:-1] + tab_set = add_quote_inside_tab(tab_set) + perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_set) + if perc_finder is not None: + match_percent = perc_finder.group(0)[1:-1] + dico_percent[tab_set] = float(match_percent) + dico_set_tab[tab_set] = ast.literal_eval(tab_set) + dico_setname_to_redis[tab_set] = set_str + else: + continue + + + message = p.get_from_set() + + while True: + + if message is not None: + filename = message + temp = filename.split('/') + timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) + content = Paste.Paste(filename).get_p_content() + + curr_set = top_termFreq_setName_day[0] + str(timestamp) + + #iterate over the words of the file + match_dico = {} + for word in content.split(): + for cur_set, array_set in dico_set_tab.items(): + for w_set in array_set[:-1]: #avoid the percent matching + if word == w_set: + try: + match_dico[str(array_set)] += 1 + except KeyError: + match_dico[str(array_set)] = 1 + + #compute matching % + for the_set, matchingNum in match_dico.items(): + eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching + if eff_percent >= dico_percent[the_set]: + + # Send a notification only when the member is in the set + if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name): + + # Send to every associated email adress + for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set): + sendEmailNotification(email, the_set) + + print(the_set, "matched in", filename) + set_name = 'set_' + dico_setname_to_redis[the_set] + new_to_the_set = server_term.sadd(set_name, filename) + new_to_the_set = True if new_to_the_set == 1 else False + + #consider the num of occurence of this set + set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1))) + + # FIXME - avoid using per paste as a set is checked over the entire paste + #1 term per paste + if new_to_the_set: + set_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_setname_to_redis[the_set], int(1))) + server_term.zincrby("per_paste_" + curr_set, dico_setname_to_redis[the_set], float(1)) + server_term.zincrby(curr_set, dico_setname_to_redis[the_set], float(1)) + + + else: + publisher.debug("Script RegexForTermsFrequency is Idling") + print "sleeping" + time.sleep(5) + message = p.get_from_set() diff --git a/bin/Curve.py b/bin/Curve.py index 14392a47..712f6006 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -35,6 +35,9 @@ import calendar from Helper import Process +# Email notifications +from NotificationHelper import * + # Config Variables BlackListTermsSet_Name = "BlackListSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet" @@ -45,7 +48,6 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] - def check_if_tracked_term(term, path): if term in server_term.smembers(TrackedTermsSet_Name): #add_paste to tracked_word_set @@ -54,6 +56,13 @@ def check_if_tracked_term(term, path): print term, 'addded', set_name, '->', path p.populate_set_out("New Term added", 'CurveManageTopSets') + # Send a notification only when the member is in the set + if term in server_term.smembers(TrackedTermsNotificationEnabled_Name): + + # Send to every associated email adress + for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): + sendEmailNotification(email, term) + def getValueOverRange(word, startDate, num_day): to_return = 0 diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py new file mode 100755 index 00000000..6d396a5d --- /dev/null +++ b/bin/NotificationHelper.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +import ConfigParser +import os +import smtplib +from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText + +""" +This module allows the global configuration and management of notification settings and methods. +""" + +# CONFIG # +configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + +# notifications enabled/disabled +TrackedTermsNotificationEnabled_Name = "TrackedNotifications" + +# associated notification email addresses for a specific term` +# Keys will be e.g. TrackedNotificationEmails +TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" + +def sendEmailNotification(recipient, term): + + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv?') + + cfg = ConfigParser.ConfigParser() + cfg.read(configfile) + + sender = cfg.get("Notifications", "sender"), + sender_host = cfg.get("Notifications", "sender_host"), + sender_port = cfg.getint("Notifications", "sender_port"), + sender_pw = cfg.get("Notifications", "sender_pw"), + + if isinstance(sender, tuple): + sender = sender[0] + + if isinstance(sender_host, tuple): + sender_host = sender_host[0] + + if isinstance(sender_port, tuple): + sender_port = sender_port[0] + + if isinstance(sender_pw, tuple): + sender_pw = sender_pw[0] + + if ( + sender is not None and + sender_host is not None and + sender_port is not None and + sender_pw is not None + ): + try: + + server_ssl = smtplib.SMTP_SSL(sender_host, sender_port) + server_ssl.ehlo() + server_ssl.login(sender, sender_pw) + + mime_msg = MIMEMultipart() + mime_msg['From'] = sender + mime_msg['To'] = recipient + mime_msg['Subject'] = "AIL Term Alert" + + body = "New occurrence for term: " + term + mime_msg.attach(MIMEText(body, 'plain')) + + server_ssl.sendmail(sender, recipient, mime_msg.as_string()) + server_ssl.quit() + + except Exception as e: + print str(e) diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py index 2efdfee5..b5570ea9 100755 --- a/bin/RegexForTermsFrequency.py +++ b/bin/RegexForTermsFrequency.py @@ -12,18 +12,22 @@ from pubsublogger import publisher from packages import lib_words from packages import Paste import os +from os import environ import datetime import calendar import re - from Helper import Process +# Email notifications +from NotificationHelper import * + # Config Variables DICO_REFRESH_TIME = 60 #s BlackListTermsSet_Name = "BlackListSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet" TrackedRegexSet_Name = "TrackedRegexSet" + top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set oneDay = 60*60*24 top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] @@ -86,8 +90,18 @@ if __name__ == "__main__": if matched is not None: #there is a match print('regex matched {}'.format(regex_str)) matched = matched.group(0) + # Add in Regex track set only if term is not in the blacklist if matched not in server_term.smembers(BlackListTermsSet_Name): + + # Send a notification only when the member is in the set + if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name): + + # Send to every associated email adress + for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched): + sendEmailNotification(email, matched) + + set_name = 'regex_' + dico_regexname_to_redis[regex_str] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py index c4e480ff..d6e9bef9 100755 --- a/bin/SetForTermsFrequency.py +++ b/bin/SetForTermsFrequency.py @@ -16,14 +16,17 @@ import datetime import calendar import re import ast - from Helper import Process +# Email notifications +from NotificationHelper import * + # Config Variables BlackListTermsSet_Name = "BlackListSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet" TrackedRegexSet_Name = "TrackedRegexSet" TrackedSetSet_Name = "TrackedSetSet" + top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set oneDay = 60*60*24 top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] @@ -100,12 +103,19 @@ if __name__ == "__main__": for the_set, matchingNum in match_dico.items(): eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching if eff_percent >= dico_percent[the_set]: + + # Send a notification only when the member is in the set + if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name): + + # Send to every associated email adress + for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set): + sendEmailNotification(email, the_set) + print(the_set, "matched in", filename) set_name = 'set_' + dico_setname_to_redis[the_set] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False - #consider the num of occurence of this set set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1))) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 7c5bcfef..2a374907 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -18,32 +18,25 @@ pystemonpath = /home/pystemon/pystemon/ sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt +##### Notifications ###### +[Notifications] +sender = sender@example.com +sender_host = smtp.example.com +sender_port = 1337 +sender_pw = securepassword + ##### Flask ##### [Flask] #Maximum number of character to display in the toolip -max_preview_char = 250 +max_preview_char = 250 #Maximum number of character to display in the modal -max_preview_modal = 800 +max_preview_modal = 800 #Default number of header to display in trending graphs default_display = 10 #Number of minutes displayed for the number of processed pastes. minute_processed_paste = 10 -#Maximum line length authorized to make a diff between duplicates -DiffMaxLineLength = 10000 - -#### Modules #### -[Categ] -#Minimum number of match between the paste and the category file -matchingThreshold=1 - -[Credential] -#Minimum length that a credential must have to be considered as such -minimumLengthThreshold=3 -#Will be pushed as alert if the number of credentials is greater to that number -criticalNumberToAlert=8 -#Will be considered as false positive if less that X matches from the top password list -minTopPassList=5 +#### Modules #### [Modules_Duplicates] #Number of month to look back maximum_month_range = 3 @@ -59,8 +52,8 @@ min_paste_size = 0.3 threshold_stucked_module=600 [Module_Mixer] -#Define the configuration of the mixer, possible value: 1, 2 or 3 -operation_mode = 3 +#Define the configuration of the mixer, possible value: 1 or 2 +operation_mode = 1 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) ttl_duplicate = 86400 @@ -153,7 +146,7 @@ maxDuplicateToPushToMISP=10 # e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557 [ZMQ_Global] #address = tcp://crf.circl.lu:5556 -address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 +address = tcp://127.0.0.1:5556 channel = 102 bind = tcp://127.0.0.1:5556 diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index e0209735..ce2fa4a7 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -42,6 +42,14 @@ TrackedRegexDate_Name = "TrackedRegexDate" TrackedSetSet_Name = "TrackedSetSet" TrackedSetDate_Name = "TrackedSetDate" +# notifications enalbed/disabled +# same value as in `bin/NotificationHelper.py` +TrackedTermsNotificationEnabled_Name = "TrackedNotifications" + +# associated notification email addresses for a specific term` +# same value as in `bin/NotificationHelper.py` +# Keys will be e.g. TrackedNotificationEmails_ +TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' @@ -138,11 +146,23 @@ def terms_management(): today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) + # Map tracking if notifications are enabled for a specific term + notificationEnabledDict = {} + + # Maps a specific term to the associated email addresses + notificationEMailTermMapping = {} + #Regex trackReg_list = [] trackReg_list_values = [] trackReg_list_num_of_paste = [] for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): + + notificationEMailTermMapping[tracked_regex] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) + + if tracked_regex not in notificationEnabledDict: + notificationEnabledDict[tracked_regex] = False + trackReg_list.append(tracked_regex) value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text) @@ -154,11 +174,21 @@ def terms_management(): value_range.append(term_date) trackReg_list_values.append(value_range) + if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): + notificationEnabledDict[tracked_regex] = True + #Set trackSet_list = [] trackSet_list_values = [] trackSet_list_num_of_paste = [] for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): + + notificationEMailTermMapping[tracked_set] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) + + + if tracked_set not in notificationEnabledDict: + notificationEnabledDict[tracked_set] = False + trackSet_list.append(tracked_set) value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text) @@ -170,11 +200,20 @@ def terms_management(): value_range.append(term_date) trackSet_list_values.append(value_range) + if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): + notificationEnabledDict[tracked_set] = True + #Tracked terms track_list = [] track_list_values = [] track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): + + notificationEMailTermMapping[tracked_term] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + + if tracked_term not in notificationEnabledDict: + notificationEnabledDict[tracked_term] = False + track_list.append(tracked_term) value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text) @@ -186,6 +225,8 @@ def terms_management(): value_range.append(term_date) track_list_values.append(value_range) + if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): + notificationEnabledDict[tracked_term] = True #blacklist terms black_list = [] @@ -199,7 +240,7 @@ def terms_management(): track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste) + per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) @terms.route("/terms_management_query_paste/") @@ -267,11 +308,36 @@ def terms_management_action(): section = request.args.get('section') action = request.args.get('action') term = request.args.get('term') + notificationEmailsParam = request.args.get('emailAddresses') + if action is None or term is None: return "None" else: if section == "followTerm": if action == "add": + + # Strip all whitespace + notificationEmailsParam = "".join(notificationEmailsParam.split()) + + # Make a list of all passed email addresses + notificationEmails = notificationEmailsParam.split(",") + + validNotificationEmails = [] + # check for valid email addresses + for email in notificationEmails: + # Really basic validation: + # has exactly one @ sign, and at least one . in the part after the @ + if re.match(r"[^@]+@[^@]+\.[^@]+", email): + validNotificationEmails.append(email) + + # add all valid emails to the set + for email in validNotificationEmails: + r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) + print "added " + email + " for " + TrackedTermsNotificationEmailsPrefix_Name + term + + # enable notifications by default + r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + # check if regex/set or simple term #regex if term.startswith('/') and term.endswith('/'): @@ -295,6 +361,16 @@ def terms_management_action(): else: r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp) + + elif action == "toggleEMailNotification": + # get the current state + if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): + # remove it + r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower()) + else: + # add it + r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + #del action else: if term.startswith('/') and term.endswith('/'): @@ -308,6 +384,9 @@ def terms_management_action(): r_serv_term.srem(TrackedTermsSet_Name, term.lower()) r_serv_term.hdel(TrackedTermsDate_Name, term.lower()) + # delete the associated notification emails too + r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) + elif section == "blacklistTerm": if action == "add": r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index a59a42a3..778b5b76 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -42,7 +42,7 @@