diff --git a/OVERVIEW.md b/OVERVIEW.md index effb387d..3d3a62ab 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -26,6 +26,24 @@ ARDB overview ARDB_DB * DB 1 - Curve * DB 2 - TermFreq + ----------------------------------------- TERM ---------------------------------------- + + SET - 'TrackedRegexSet' term + + HSET - 'TrackedRegexDate' tracked_regex today_timestamp + + SET - 'TrackedSetSet' set_to_add + + HSET - 'TrackedSetDate' set_to_add today_timestamp + + SET - 'TrackedSetTermSet' term + + HSET - 'TrackedTermDate' tracked_regex today_timestamp + + SET - 'TrackedNotificationEmails_'+term/set email + + SET - 'TrackedNotifications' term/set + * DB 3 - Trending * DB 4 - Sentiment * DB 5 - TermCred diff --git a/bin/Curve.py b/bin/Curve.py index 8e228039..c7083c54 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -71,6 +73,11 @@ def check_if_tracked_term(term, path): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term): + msg = '{};{}'.format(tag, path) + p.populate_set_out(msg, 'Tags') + def getValueOverRange(word, startDate, num_day): to_return = 0 diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py index fae7a03a..0db7f2ee 100755 --- a/bin/RegexForTermsFrequency.py +++ b/bin/RegexForTermsFrequency.py @@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -129,6 +131,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + set_name = 'regex_' + dico_regexname_to_redis[regex_str] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py index 78de9b08..19ed7210 100755 --- a/bin/SetForTermsFrequency.py +++ b/bin/SetForTermsFrequency.py @@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -121,6 +123,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + print(the_set, "matched in", filename) set_name = 'set_' + dico_setname_to_redis[the_set] new_to_the_set = server_term.sadd(set_name, filename) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index deb5a069..0dc40448 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -32,13 +32,15 @@ publish = Redis_Words [Curve] subscribe = Redis_Words -publish = Redis_CurveManageTopSets +publish = Redis_CurveManageTopSets,Redis_Tags [RegexForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [SetForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [CurveManageTopSets] subscribe = Redis_CurveManageTopSets diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index ea6fd6ed..7cc802f0 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis( db=cfg.getint("ARDB_Onion", "db"), decode_responses=True) - sys.path.append('../../configs/keys') # MISP # try: diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 16930ef8..cc38de77 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -506,6 +506,8 @@ def edit_tag_export(): status_misp = [] status_hive = [] + infoleak_tags = Taxonomies().get('infoleak').machinetags() + is_infoleak_tag = [] for tag in list_export_tags: if r_serv_db.sismember('whitelist_misp', tag): @@ -519,6 +521,11 @@ def edit_tag_export(): else: status_hive.append(False) + if tag in infoleak_tags: + is_infoleak_tag.append(True) + else: + is_infoleak_tag.append(False) + if misp_auto_events is not None: if int(misp_auto_events) == 1: misp_active = True @@ -543,6 +550,7 @@ def edit_tag_export(): misp_active=misp_active, hive_active=hive_active, list_export_tags=list_export_tags, + is_infoleak_tag=is_infoleak_tag, status_misp=status_misp, status_hive=status_hive, nb_tags_whitelist_misp=nb_tags_whitelist_misp, @@ -594,5 +602,37 @@ def disable_hive_auto_alert(): r_serv_db.set('hive:auto-alerts', 0) return edit_tag_export() +@PasteSubmit.route("/PasteSubmit/add_push_tag") +def add_push_tag(): + tag = request.args.get('tag') + if tag is not None: + + #limit tag length + if len(tag) > 49: + tag = tag[0:48] + + r_serv_db.sadd('list_export_tags', tag) + + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'None args', 400 + +@PasteSubmit.route("/PasteSubmit/delete_push_tag") +def delete_push_tag(): + tag = request.args.get('tag') + + infoleak_tags = Taxonomies().get('infoleak').machinetags() + if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): + r_serv_db.srem('list_export_tags', tag) + r_serv_db.srem('whitelist_misp', tag) + r_serv_db.srem('whitelist_hive', tag) + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'this tag can\'t be removed', 400 + # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit, url_prefix=baseUrl) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html index 04a506d6..94980787 100644 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -37,6 +37,9 @@ background: #d91f2d; color: #fff; } + .mouse_pointer{ + cursor: pointer; + } @@ -169,7 +172,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -209,7 +219,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -232,6 +249,42 @@ +
+ + + + +
+ @@ -277,6 +330,25 @@ $(document).ready(function(){ } ); } + + function delete_push_tag(tag){ + //var row_tr = $(this).closest("tr"); + $.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } + + function add_custom_tag(){ + $.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index b0794593..86d67af8 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -10,7 +10,7 @@ import redis import datetime import calendar import flask -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect import re import Paste from pprint import pprint @@ -24,6 +24,8 @@ cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred +r_serv_db = Flask_config.r_serv_db +bootstrap_label = Flask_config.bootstrap_label terms = Blueprint('terms', __name__, template_folder='templates') @@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # same value as in `bin/NotificationHelper.py` # Keys will be e.g. TrackedNotificationEmails_ TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' @@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False): filtered_usernames.append(usr) return filtered_usernames +def save_tag_to_auto_push(list_tag): + for tag in set(list_tag): + #limit tag length + if len(tag) > 49: + tag = tag[0:48] + r_serv_db.sadd('list_export_tags', tag) # ============ ROUTES ============ @@ -152,6 +161,7 @@ def terms_management(): # Maps a specific term to the associated email addresses notificationEMailTermMapping = {} + notificationTagsTermMapping = {} #Regex trackReg_list = [] @@ -159,7 +169,8 @@ def terms_management(): trackReg_list_num_of_paste = [] for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): - notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) + notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex) + notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) if tracked_regex not in notificationEnabledDict: notificationEnabledDict[tracked_regex] = False @@ -185,8 +196,8 @@ def terms_management(): for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): tracked_set = tracked_set - notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) - + notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set) + notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) if tracked_set not in notificationEnabledDict: notificationEnabledDict[tracked_set] = False @@ -211,7 +222,8 @@ def terms_management(): track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): - notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term) + notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) if tracked_term not in notificationEnabledDict: notificationEnabledDict[tracked_term] = False @@ -244,7 +256,8 @@ def terms_management(): track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) + per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label, + notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping) @terms.route("/terms_management_query_paste/") @@ -313,6 +326,7 @@ def terms_management_action(): action = request.args.get('action') term = request.args.get('term') notificationEmailsParam = request.args.get('emailAddresses') + input_tags = request.args.get('tags') if action is None or term is None or notificationEmailsParam is None: return "None" @@ -320,11 +334,8 @@ def terms_management_action(): if section == "followTerm": if action == "add": - # Strip all whitespace - notificationEmailsParam = "".join(notificationEmailsParam.split()) - # Make a list of all passed email addresses - notificationEmails = notificationEmailsParam.split(",") + notificationEmails = notificationEmailsParam.split() validNotificationEmails = [] # check for valid email addresses @@ -334,6 +345,8 @@ def terms_management_action(): if re.match(r"[^@]+@[^@]+\.[^@]+", email): validNotificationEmails.append(email) + # create tags list + list_tags = input_tags.split() # check if regex/set or simple term #regex @@ -345,6 +358,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) + save_tag_to_auto_push(list_tags) #set elif term.startswith('\\') and term.endswith('\\'): @@ -363,6 +380,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) + save_tag_to_auto_push(list_tags) #simple term else: @@ -373,6 +394,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) + save_tag_to_auto_push(list_tags) elif action == "toggleEMailNotification": # get the current state @@ -397,6 +422,8 @@ def terms_management_action(): # delete the associated notification emails too r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) + # delete the associated tags set + r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term) elif section == "blacklistTerm": if action == "add": @@ -413,6 +440,28 @@ def terms_management_action(): to_return["term"] = term return jsonify(to_return) +@terms.route("/terms_management/delete_terms_tags", methods=['POST']) +def delete_terms_tags(): + term = request.form.get('term') + tags_to_delete = request.form.getlist('tags_to_delete') + + if term is not None and tags_to_delete is not None: + for tag in tags_to_delete: + r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag) + return redirect(url_for('terms.terms_management')) + else: + return 'None args', 400 + +@terms.route("/terms_management/delete_terms_email", methods=['GET']) +def delete_terms_email(): + term = request.args.get('term') + email = request.args.get('email') + + if term is not None and email is not None: + r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email) + return redirect(url_for('terms.terms_management')) + else: + return 'None args', 400 @terms.route("/terms_plot_tool/") diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 0efda575..a2ecd906 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -36,6 +36,12 @@ white-space:pre-wrap; word-wrap:break-word; } + .mouse_pointer{ + cursor: pointer; + } + .lb-md { + font-size: 16px; + } @@ -74,6 +80,7 @@
+ {% set uniq_id = namespace(modal_id=0)%}