From dad0365c67baf6e936f610f26ded1e5e87f2b103 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 09:16:18 +0100 Subject: [PATCH 01/61] fix:[MISP_The_Hive_feeder] set default values --- bin/MISP_The_Hive_feeder.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 0a8f1791..234f976a 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -119,10 +119,18 @@ if __name__ == "__main__": db=cfg.getint("ARDB_Metadata", "db"), decode_responses=True) + # set sensor uuid uuid_ail = r_serv_db.get('ail:uuid') if uuid_ail is None: uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() ) + # set default + if r_serv_db.get('hive:auto-alerts') is None: + r_serv_db.set('hive:auto-alerts', 0) + + if r_serv_db.get('misp:auto-events') is None: + r_serv_db.set('misp:auto-events', 0): + p = Process(config_section) # create MISP connection if flag_misp: @@ -191,7 +199,6 @@ if __name__ == "__main__": whitelist_hive = r_serv_db.scard('whitelist_hive') if r_serv_db.sismember('whitelist_hive', tag): create_the_hive_alert(source, path, full_path, tag) - else: print('hive, auto alerts creation disable') if flag_misp: From 5c67297dc8cda04f92b9766b9a0c562f1246a6f3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 10:13:48 +0100 Subject: [PATCH 02/61] fix: [Search] Improve search performance note: search by paste name is temporary disabled --- var/www/modules/search/Flask_search.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 5b458589..51735bf3 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -16,6 +16,8 @@ from whoosh import index from whoosh.fields import Schema, TEXT, ID from whoosh.qparser import QueryParser +import time + # ============ VARIABLES ============ import Flask_config @@ -55,8 +57,8 @@ def get_index_list(selected_index=""): if os.path.isdir(os.path.join(baseindexpath, dirs)): value = dirs name = to_iso_date(dirs) + " - " + \ - str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \ - "(" + str(get_item_count(dirs)) + " Items" + ")" + str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \ + #"(" + str(get_item_count(dirs))''' + " Items" + ")" flag = dirs==selected_index.split('/')[-1] if dirs == "old_index": temp = [value, name, flag] @@ -66,6 +68,7 @@ def get_index_list(selected_index=""): index_list.sort(reverse=True, key=lambda x: x[0]) if len(temp) != 0: index_list.append(temp) + return index_list def get_dir_size(directory): @@ -108,6 +111,7 @@ def search(): else: selected_index = os.path.join(baseindexpath, index_name) + ''' temporary disabled # Search filename for path in r_serv_pasteName.smembers(q[0]): r.append(path) @@ -119,13 +123,14 @@ def search(): curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) + ''' # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: - query = QueryParser("content", ix.schema).parse(" ".join(q)) + query = QueryParser("content", ix.schema).parse("".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1]) @@ -159,15 +164,18 @@ def search(): results = searcher.search(query) num_res = len(results) + index_list = get_index_list() + index_min = 1 - index_max = len(get_index_list()) + index_max = len(index_list) + return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res, index_min=index_min, index_max=index_max, bootstrap_label=bootstrap_label, paste_tags=paste_tags, - index_list=get_index_list(selected_index) + index_list=index_list ) From af1ff6225ed77a5ea14a5f10403a9aa8f830657f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 10:36:58 +0100 Subject: [PATCH 03/61] fix: [Search] tags display --- var/www/modules/search/Flask_search.py | 3 ++- var/www/modules/search/templates/search.html | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 51735bf3..7f6cd724 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -220,6 +220,7 @@ def get_more_search_result(): p_tags = r_serv_metadata.smembers('tag:'+path) l_tags = [] for tag in p_tags: + complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': @@ -231,7 +232,7 @@ def get_more_search_result(): else: tag = tag[0] - l_tags.append(tag) + l_tags.append( (tag, complete_tag) ) list_tags.append(l_tags) to_return = {} diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index adc1b555..5754a0ee 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -201,7 +201,9 @@ var curr_preview = data.preview_array[i].replace(/\"/g, "\'"); var tag = "" for(j=0; j" + data.list_tags[j] + "" + tag = tag + "" + + "" + data.list_tags[i][j][0] + + "" + "" } search_table.row.add( [ init_num_of_elements_in_table+((offset))+i+1, From aa16c52cbf1ce24624f6fd676b5e296c49a988da Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 14:20:12 +0100 Subject: [PATCH 04/61] fix: [NotificationHelper] add config --- bin/NotificationHelper.py | 22 ++++++---------------- bin/packages/config.cfg.sample | 1 + 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index d8f7fe92..1e3ab8a0 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -33,22 +33,12 @@ def sendEmailNotification(recipient, alert_name, content): cfg = configparser.ConfigParser() cfg.read(configfile) - sender = cfg.get("Notifications", "sender"), - sender_host = cfg.get("Notifications", "sender_host"), - sender_port = cfg.getint("Notifications", "sender_port"), - sender_pw = cfg.get("Notifications", "sender_pw"), - - if isinstance(sender, tuple): - sender = sender[0] - - if isinstance(sender_host, tuple): - sender_host = sender_host[0] - - if isinstance(sender_port, tuple): - sender_port = sender_port[0] - - if isinstance(sender_pw, tuple): - sender_pw = sender_pw[0] + sender = cfg.get("Notifications", "sender") + sender_host = cfg.get("Notifications", "sender_host") + sender_port = cfg.getint("Notifications", "sender_port") + sender_pw = cfg.get("Notifications", "sender_pw") + if sender_pw = 'None': + sender_pw = None # raise an exception if any of these is None if (sender is None or diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index fbe4f6f3..97cd692a 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -27,6 +27,7 @@ ail_domain = http://localhost:7000 sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 +sender_pw = None # optional for using with authenticated SMTP over SSL # sender_pw = securepassword From 877ac0636ea1eaa0eae7ac4552804407bdb13785 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 14:30:03 +0100 Subject: [PATCH 05/61] fix: typo --- bin/NotificationHelper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index 1e3ab8a0..703f46e8 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -37,7 +37,7 @@ def sendEmailNotification(recipient, alert_name, content): sender_host = cfg.get("Notifications", "sender_host") sender_port = cfg.getint("Notifications", "sender_port") sender_pw = cfg.get("Notifications", "sender_pw") - if sender_pw = 'None': + if sender_pw == 'None': sender_pw = None # raise an exception if any of these is None From 85f933fd810f913dad185bb5cf58140e8d369ae3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 14:59:40 +0100 Subject: [PATCH 06/61] chg: [NotificationHelper] add error traceback --- bin/NotificationHelper.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index 703f46e8..1bccd314 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -3,8 +3,10 @@ import argparse import configparser +import traceback import os import smtplib +from pubsublogger import publisher from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') +publisher.port = 6380 +publisher.channel = "Script" + # notifications enabled/disabled TrackedTermsNotificationEnabled_Name = "TrackedNotifications" @@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # Keys will be e.g. TrackedNotificationEmails TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" - def sendEmailNotification(recipient, alert_name, content): if not os.path.exists(configfile): @@ -73,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content): smtp_server.quit() print('Send notification ' + alert_name + ' to '+recipient) - except Exception as e: - print(str(e)) - # raise e + except Exception as err: + traceback.print_tb(err.__traceback__) + publisher.warning(err) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Test notification sender.') From 7ef5cc3205306837caad12625e468b9a571cec9b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 5 Nov 2018 16:40:25 +0100 Subject: [PATCH 07/61] fix:[MISP_The_Hive_feeder] typo --- bin/MISP_The_Hive_feeder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 234f976a..4d471ff2 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -129,7 +129,7 @@ if __name__ == "__main__": r_serv_db.set('hive:auto-alerts', 0) if r_serv_db.get('misp:auto-events') is None: - r_serv_db.set('misp:auto-events', 0): + r_serv_db.set('misp:auto-events', 0) p = Process(config_section) # create MISP connection From 979472df15b0a20b3e11f35a9f5c9d0b82e2c6d7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 6 Nov 2018 13:38:37 +0100 Subject: [PATCH 08/61] chg: [Terms] tag tracked terms --- OVERVIEW.md | 18 +++++++ bin/Curve.py | 7 +++ bin/RegexForTermsFrequency.py | 7 +++ bin/SetForTermsFrequency.py | 7 +++ bin/packages/modules.cfg | 4 +- var/www/modules/terms/Flask_terms.py | 25 ++++++++- .../terms/templates/terms_management.html | 51 ++++++++++++++----- 7 files changed, 103 insertions(+), 16 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index effb387d..3d3a62ab 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -26,6 +26,24 @@ ARDB overview ARDB_DB * DB 1 - Curve * DB 2 - TermFreq + ----------------------------------------- TERM ---------------------------------------- + + SET - 'TrackedRegexSet' term + + HSET - 'TrackedRegexDate' tracked_regex today_timestamp + + SET - 'TrackedSetSet' set_to_add + + HSET - 'TrackedSetDate' set_to_add today_timestamp + + SET - 'TrackedSetTermSet' term + + HSET - 'TrackedTermDate' tracked_regex today_timestamp + + SET - 'TrackedNotificationEmails_'+term/set email + + SET - 'TrackedNotifications' term/set + * DB 3 - Trending * DB 4 - Sentiment * DB 5 - TermCred diff --git a/bin/Curve.py b/bin/Curve.py index 8e228039..c7083c54 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -71,6 +73,11 @@ def check_if_tracked_term(term, path): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term): + msg = '{};{}'.format(tag, path) + p.populate_set_out(msg, 'Tags') + def getValueOverRange(word, startDate, num_day): to_return = 0 diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py index fae7a03a..0db7f2ee 100755 --- a/bin/RegexForTermsFrequency.py +++ b/bin/RegexForTermsFrequency.py @@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -129,6 +131,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + set_name = 'regex_' + dico_regexname_to_redis[regex_str] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py index 78de9b08..19ed7210 100755 --- a/bin/SetForTermsFrequency.py +++ b/bin/SetForTermsFrequency.py @@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -121,6 +123,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + print(the_set, "matched in", filename) set_name = 'set_' + dico_setname_to_redis[the_set] new_to_the_set = server_term.sadd(set_name, filename) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index deb5a069..0dc40448 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -32,13 +32,15 @@ publish = Redis_Words [Curve] subscribe = Redis_Words -publish = Redis_CurveManageTopSets +publish = Redis_CurveManageTopSets,Redis_Tags [RegexForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [SetForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [CurveManageTopSets] subscribe = Redis_CurveManageTopSets diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index b0794593..f489498d 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -24,6 +24,7 @@ cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred +bootstrap_label = Flask_config.bootstrap_label terms = Blueprint('terms', __name__, template_folder='templates') @@ -51,6 +52,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # same value as in `bin/NotificationHelper.py` # Keys will be e.g. TrackedNotificationEmails_ TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' @@ -152,6 +154,7 @@ def terms_management(): # Maps a specific term to the associated email addresses notificationEMailTermMapping = {} + notificationTagsTermMapping = {} #Regex trackReg_list = [] @@ -160,6 +163,7 @@ def terms_management(): for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) + notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) if tracked_regex not in notificationEnabledDict: notificationEnabledDict[tracked_regex] = False @@ -186,7 +190,7 @@ def terms_management(): tracked_set = tracked_set notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) - + notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) if tracked_set not in notificationEnabledDict: notificationEnabledDict[tracked_set] = False @@ -212,6 +216,7 @@ def terms_management(): for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) if tracked_term not in notificationEnabledDict: notificationEnabledDict[tracked_term] = False @@ -239,12 +244,14 @@ def terms_management(): term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" black_list.append([blacked_term, term_date]) + print(notificationTagsTermMapping) return render_template("terms_management.html", black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) + per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label, + notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping) @terms.route("/terms_management_query_paste/") @@ -313,6 +320,7 @@ def terms_management_action(): action = request.args.get('action') term = request.args.get('term') notificationEmailsParam = request.args.get('emailAddresses') + input_tags = request.args.get('tags') if action is None or term is None or notificationEmailsParam is None: return "None" @@ -334,6 +342,8 @@ def terms_management_action(): if re.match(r"[^@]+@[^@]+\.[^@]+", email): validNotificationEmails.append(email) + # create tags list + list_tags = input_tags.split() # check if regex/set or simple term #regex @@ -345,6 +355,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) #set elif term.startswith('\\') and term.endswith('\\'): @@ -363,6 +376,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) #simple term else: @@ -373,6 +389,9 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) elif action == "toggleEMailNotification": # get the current state @@ -397,6 +416,8 @@ def terms_management_action(): # delete the associated notification emails too r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) + # delete the associated tags set + r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term) elif section == "blacklistTerm": if action == "add": diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 0efda575..fc92946a 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -98,7 +98,8 @@
- + +
@@ -119,7 +120,16 @@ {% for set in trackSet_list %} - {{ set }} + + {{ set }} +
+ {% for tag in notificationTagsTermMapping[set] %} + + {{ tag }} + + {% endfor %} +
+ {{ trackSet_list_values[loop.index0][3] }} {{ trackSet_list_values[loop.index0][0] }} {{ trackSet_list_values[loop.index0][1] }} @@ -136,7 +146,16 @@ {% for regex in trackReg_list %} - {{ regex }} + + {{ regex }} +
+ {% for tag in notificationTagsTermMapping[regex] %} + + {{ tag }} + + {% endfor %} +
+ {{ trackReg_list_values[loop.index0][3] }} {{ trackReg_list_values[loop.index0][0] }} {{ trackReg_list_values[loop.index0][1] }} @@ -153,7 +172,16 @@ {% for term in track_list %} - {{ term }} + + {{ term }} +
+ {% for tag in notificationTagsTermMapping[term] %} + + {{ tag }} + + {% endfor %} +
+ {{ track_list_values[loop.index0][3] }} {{ track_list_values[loop.index0][0] }} {{ track_list_values[loop.index0][1] }} @@ -351,17 +379,19 @@ function perform_binding() { function perform_operation(){ var curr_section = $(this).attr('data-section'); var curr_action = $(this).attr('data-action'); + var row_tr = $(this).closest("tr"); if (curr_action == "add") { var curr_term = $('#'+curr_section+'Input').val(); var email_addresses = $('#followTermEMailNotificationReceiversInput').val(); + var tags = $('#followTermTag').val(); } else { var curr_term = $(this).attr('data-content'); var email_addresses = ""; } - var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses}; + var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags}; if (curr_term != "") { - console.log(data_to_send); + //console.log(data_to_send); $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){ if(status == "success") { var json = data; @@ -372,13 +402,8 @@ function perform_operation(){ $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ reload_per_paste(); }); - } else if (json.action == "delete") { - // Find indexes of row which have the term in the first column - var index = table_track.rows().eq( 0 ).filter( function (rowIdx) { - console.log(table_track.cell( rowIdx, 0 ).data()) - return table_track.cell( rowIdx, 0 ).data() === json.term; - } ); - table_track.rows(index).remove().draw( false ); + } else if (json.action == "delete") { + row_tr.remove() } } else if(json.section == "blacklistTerm"){ if(json.action == "add") { From a3fa5a4dcfa0114dc656f609facebaf0c4392822 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 6 Nov 2018 16:08:58 +0100 Subject: [PATCH 09/61] chg: [auto push tag] delete unwanted tags --- var/www/modules/Flask_config.py | 1 - .../modules/PasteSubmit/Flask_PasteSubmit.py | 22 +++++++++++++++++++ .../templates/edit_tag_export.html | 22 ++++++++++++++++++- var/www/modules/terms/Flask_terms.py | 8 ++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index ea6fd6ed..7cc802f0 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis( db=cfg.getint("ARDB_Onion", "db"), decode_responses=True) - sys.path.append('../../configs/keys') # MISP # try: diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 16930ef8..5e26d941 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -506,6 +506,8 @@ def edit_tag_export(): status_misp = [] status_hive = [] + infoleak_tags = Taxonomies().get('infoleak').machinetags() + is_infoleak_tag = [] for tag in list_export_tags: if r_serv_db.sismember('whitelist_misp', tag): @@ -519,6 +521,11 @@ def edit_tag_export(): else: status_hive.append(False) + if tag in infoleak_tags: + is_infoleak_tag.append(True) + else: + is_infoleak_tag.append(False) + if misp_auto_events is not None: if int(misp_auto_events) == 1: misp_active = True @@ -543,6 +550,7 @@ def edit_tag_export(): misp_active=misp_active, hive_active=hive_active, list_export_tags=list_export_tags, + is_infoleak_tag=is_infoleak_tag, status_misp=status_misp, status_hive=status_hive, nb_tags_whitelist_misp=nb_tags_whitelist_misp, @@ -594,5 +602,19 @@ def disable_hive_auto_alert(): r_serv_db.set('hive:auto-alerts', 0) return edit_tag_export() +@PasteSubmit.route("/PasteSubmit/delete_push_tag") +def delete_push_tag(): + tag = request.args.get('tag') + + infoleak_tags = Taxonomies().get('infoleak').machinetags() + if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): + r_serv_db.srem('list_export_tags', tag) + #print('deleted') + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'this tag can\'t be removed', 400 + # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit, url_prefix=baseUrl) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html index 04a506d6..74b24c56 100644 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -37,6 +37,9 @@ background: #d91f2d; color: #fff; } + .mouse_pointer{ + cursor: pointer; + } @@ -169,7 +172,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -277,6 +287,16 @@ $(document).ready(function(){ } ); } + + function delete_push_tag(tag){ + //var row_tr = $(this).closest("tr"); + $.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index f489498d..c4aebc08 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -24,6 +24,7 @@ cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred +r_serv_db = Flask_config.r_serv_db bootstrap_label = Flask_config.bootstrap_label terms = Blueprint('terms', __name__, template_folder='templates') @@ -132,6 +133,9 @@ def mixUserName(supplied, extensive=False): filtered_usernames.append(usr) return filtered_usernames +def save_tag_to_auto_push(list_tag): + for tag in set(list_tag): + r_serv_db.sadd('list_export_tags', tag) # ============ ROUTES ============ @@ -244,7 +248,6 @@ def terms_management(): term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" black_list.append([blacked_term, term_date]) - print(notificationTagsTermMapping) return render_template("terms_management.html", black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, @@ -358,6 +361,7 @@ def terms_management_action(): # add tags list for tag in list_tags: r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) + save_tag_to_auto_push(list_tags) #set elif term.startswith('\\') and term.endswith('\\'): @@ -379,6 +383,7 @@ def terms_management_action(): # add tags list for tag in list_tags: r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) + save_tag_to_auto_push(list_tags) #simple term else: @@ -392,6 +397,7 @@ def terms_management_action(): # add tags list for tag in list_tags: r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) + save_tag_to_auto_push(list_tags) elif action == "toggleEMailNotification": # get the current state From 0ad80a22ca2f10577c54030181cbd5b141bc7e23 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 6 Nov 2018 17:04:02 +0100 Subject: [PATCH 10/61] fix: [update_thirdparty] upgrade github source --- var/www/update_thirdparty.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 76ae9174..47fd4ecc 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -86,11 +86,11 @@ popd #active virtualenv source ./../../AILENV/bin/activate #Update MISP Taxonomies and Galaxies -python3 -m pip install git+https://github.com/MISP/PyTaxonomies -python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies +python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade +python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade #Update PyMISP -python3 -m pip install git+https://github.com/MISP/PyMISP +python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade #Update the Hive -python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py +python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py --upgrade From f91aba5f6f5fc5c3b093f84c3246258d3c3cc4fe Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 7 Nov 2018 09:41:58 +0100 Subject: [PATCH 11/61] chg: [tag export] UI, add custom tags --- .../modules/PasteSubmit/Flask_PasteSubmit.py | 12 ++++- .../templates/edit_tag_export.html | 54 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 5e26d941..47f6286b 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -602,6 +602,15 @@ def disable_hive_auto_alert(): r_serv_db.set('hive:auto-alerts', 0) return edit_tag_export() +@PasteSubmit.route("/PasteSubmit/add_push_tag") +def add_push_tag(): + tag = request.args.get('tag') + r_serv_db.sadd('list_export_tags', tag) + + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + @PasteSubmit.route("/PasteSubmit/delete_push_tag") def delete_push_tag(): tag = request.args.get('tag') @@ -609,7 +618,8 @@ def delete_push_tag(): infoleak_tags = Taxonomies().get('infoleak').machinetags() if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): r_serv_db.srem('list_export_tags', tag) - #print('deleted') + r_serv_db.srem('whitelist_misp', tag) + r_serv_db.srem('whitelist_hive', tag) to_return = {} to_return["tag"] = tag return jsonify(to_return) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html index 74b24c56..94980787 100644 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -219,7 +219,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -242,6 +249,42 @@ +
+ + + + +
+ @@ -297,6 +340,15 @@ $(document).ready(function(){ } }); } + + function add_custom_tag(){ + $.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } From 2c46c7f2abbc496a921694f87f309ec5dfc4c09c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 7 Nov 2018 10:17:57 +0100 Subject: [PATCH 12/61] fix: [tag export] limit custom tag lenght --- var/www/modules/PasteSubmit/Flask_PasteSubmit.py | 4 ++++ var/www/modules/terms/Flask_terms.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 47f6286b..79aea450 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -605,6 +605,10 @@ def disable_hive_auto_alert(): @PasteSubmit.route("/PasteSubmit/add_push_tag") def add_push_tag(): tag = request.args.get('tag') + #limit tag length + if len(tag) > 49: + tag = tag[0:48] + r_serv_db.sadd('list_export_tags', tag) to_return = {} diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index c4aebc08..1f0bbfb3 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -135,6 +135,9 @@ def mixUserName(supplied, extensive=False): def save_tag_to_auto_push(list_tag): for tag in set(list_tag): + #limit tag length + if len(tag) > 49: + tag = tag[0:48] r_serv_db.sadd('list_export_tags', tag) # ============ ROUTES ============ From 46614c97c824b0a5e1d3dc492e8fabd52426cd81 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 7 Nov 2018 11:42:31 +0100 Subject: [PATCH 13/61] chg: [terms_management] delete email --- .../modules/PasteSubmit/Flask_PasteSubmit.py | 18 +++++---- var/www/modules/terms/Flask_terms.py | 21 +++++++---- .../terms/templates/terms_management.html | 37 +++++++++++++++++-- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 79aea450..cc38de77 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -605,15 +605,19 @@ def disable_hive_auto_alert(): @PasteSubmit.route("/PasteSubmit/add_push_tag") def add_push_tag(): tag = request.args.get('tag') - #limit tag length - if len(tag) > 49: - tag = tag[0:48] + if tag is not None: - r_serv_db.sadd('list_export_tags', tag) + #limit tag length + if len(tag) > 49: + tag = tag[0:48] - to_return = {} - to_return["tag"] = tag - return jsonify(to_return) + r_serv_db.sadd('list_export_tags', tag) + + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'None args', 400 @PasteSubmit.route("/PasteSubmit/delete_push_tag") def delete_push_tag(): diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 1f0bbfb3..d1a067da 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -169,7 +169,7 @@ def terms_management(): trackReg_list_num_of_paste = [] for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): - notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) + notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex) notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) if tracked_regex not in notificationEnabledDict: @@ -196,7 +196,7 @@ def terms_management(): for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): tracked_set = tracked_set - notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) + notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set) notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) if tracked_set not in notificationEnabledDict: @@ -222,7 +222,7 @@ def terms_management(): track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): - notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term) notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) if tracked_term not in notificationEnabledDict: @@ -334,11 +334,8 @@ def terms_management_action(): if section == "followTerm": if action == "add": - # Strip all whitespace - notificationEmailsParam = "".join(notificationEmailsParam.split()) - # Make a list of all passed email addresses - notificationEmails = notificationEmailsParam.split(",") + notificationEmails = notificationEmailsParam.split() validNotificationEmails = [] # check for valid email addresses @@ -443,6 +440,16 @@ def terms_management_action(): to_return["term"] = term return jsonify(to_return) +@terms.route("/terms_management/delete_terms_email", methods=['GET']) +def delete_terms_email(): + term = request.args.get('term') + email = request.args.get('email') + + if term is not None and email is not None: + r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email) + return 'sucess' + else: + return 'None args', 400 @terms.route("/terms_plot_tool/") diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index fc92946a..6acc5735 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -36,6 +36,9 @@ white-space:pre-wrap; word-wrap:break-word; } + .mouse_pointer{ + cursor: pointer; + } @@ -98,7 +101,7 @@
- +
@@ -140,7 +143,13 @@    

- {{ notificationEMailTermMapping[set] }} + + {% for email in notificationEMailTermMapping[set] %} +
+ {{ email }} +
+ {% endfor %} + {% endfor %} @@ -166,7 +175,13 @@    

- {{ notificationEMailTermMapping[regex] }} + + {% for email in notificationEMailTermMapping[regex] %} +
+ {{ email }} +
+ {% endfor %} + {% endfor %} @@ -192,7 +207,13 @@    

- {{ notificationEMailTermMapping[term] }} + + {% for email in notificationEMailTermMapping[term] %} +
+ {{ email }} +
+ {% endfor %} + {% endfor %} @@ -428,4 +449,12 @@ function perform_operation(){ }); } } + +function delete_email(term, email){ + $.get("{{ url_for('terms.delete_terms_email') }}", { term: term, email: email }, function(data, status){ + if(status == "success") { + window.location.reload(false); + } + }); +} From 0adb8b4624583d5a570730c04022f367475a6e5b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 7 Nov 2018 15:37:25 +0100 Subject: [PATCH 14/61] chg: [terms_management] delete terms tags --- var/www/modules/terms/Flask_terms.py | 14 +- .../terms/templates/terms_management.html | 127 ++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index d1a067da..e6bb9fbc 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -10,7 +10,7 @@ import redis import datetime import calendar import flask -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect import re import Paste from pprint import pprint @@ -440,6 +440,18 @@ def terms_management_action(): to_return["term"] = term return jsonify(to_return) +@terms.route("/terms_management/delete_terms_tags", methods=['POST']) +def delete_terms_tags(): + term = request.form.get('term') + tags_to_delete = request.form.getlist('tags_to_delete') + + if term is not None and tags_to_delete is not None: + for tag in tags_to_delete: + r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag) + return redirect(url_for('terms.terms_management')) + else: + return 'None args', 400 + @terms.route("/terms_management/delete_terms_email", methods=['GET']) def delete_terms_email(): term = request.args.get('term') diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 6acc5735..29bbfad5 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -39,6 +39,9 @@ .mouse_pointer{ cursor: pointer; } + .lb-md { + font-size: 16px; + } @@ -77,6 +80,7 @@
+ {% set uniq_id = 0 %}
{{ trackSet_list_values[loop.index0][3] }} @@ -163,6 +208,47 @@ {{ tag }} {% endfor %} + {% if notificationTagsTermMapping[regex] %} +
+ + + {% set uniq_id = uniq_id + 1 %} + {% endif %}
{{ trackReg_list_values[loop.index0][3] }} @@ -195,6 +281,47 @@ {{ tag }} {% endfor %} + {% if notificationTagsTermMapping[term] %} +
+ + + {% set uniq_id = uniq_id + 1 %} + {% endif %}
{{ track_list_values[loop.index0][3] }} From 4e906ff841598bbf2d862def41a30caf5367a192 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 8 Nov 2018 10:41:37 +0100 Subject: [PATCH 15/61] fix: [term_management] delete emails --- var/www/modules/terms/Flask_terms.py | 2 +- .../terms/templates/terms_management.html | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index e6bb9fbc..86d67af8 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -459,7 +459,7 @@ def delete_terms_email(): if term is not None and email is not None: r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email) - return 'sucess' + return redirect(url_for('terms.terms_management')) else: return 'None args', 400 diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 29bbfad5..b84b7042 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -190,7 +190,11 @@

{% for email in notificationEMailTermMapping[set] %} -
+ +
+ +
+ {{ email }}
{% endfor %} @@ -263,7 +267,11 @@

{% for email in notificationEMailTermMapping[regex] %} -
+ +
+ +
+ {{ email }}
{% endfor %} @@ -336,7 +344,11 @@

{% for email in notificationEMailTermMapping[term] %} -
+ +
+ +
+ {{ email }}
{% endfor %} @@ -576,12 +588,4 @@ function perform_operation(){ }); } } - -function delete_email(term, email){ - $.get("{{ url_for('terms.delete_terms_email') }}", { term: term, email: email }, function(data, status){ - if(status == "success") { - window.location.reload(false); - } - }); -} From e3764ca5755c3bc2667137d62a526a85ec8e280e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 8 Nov 2018 11:22:23 +0100 Subject: [PATCH 16/61] fix: [MISP_the_Hive_feeder] force absolute path --- bin/MISP_The_Hive_feeder.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 4d471ff2..ff6552f2 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper -def create_the_hive_alert(source, path, content, tag): +def create_the_hive_alert(source, path, tag): tags = list(r_serv_metadata.smembers('tag:'+path)) artifacts = [ @@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag): ] l_tags = tag.split(',') - print(tag) # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] @@ -175,6 +174,9 @@ if __name__ == "__main__": r_serv_db.set('ail:thehive', False) print('Not connected to The HIVE') + ## FIXME: remove it + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + while True: # Get one message from the input queue @@ -187,18 +189,17 @@ if __name__ == "__main__": if flag_the_hive or flag_misp: tag, path = message.split(';') + ## FIXME: remove it + if PASTES_FOLDER not in path: + path = os.path.join(PASTES_FOLDER, path) paste = Paste.Paste(path) source = '/'.join(paste.p_path.split('/')[-6:]) - full_path = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "pastes"), path) - - if HiveApi != False: if int(r_serv_db.get('hive:auto-alerts')) == 1: whitelist_hive = r_serv_db.scard('whitelist_hive') if r_serv_db.sismember('whitelist_hive', tag): - create_the_hive_alert(source, path, full_path, tag) + create_the_hive_alert(source, path, tag) else: print('hive, auto alerts creation disable') if flag_misp: From 173c0a15c61265d7392b5a9e4fa4e14d800814fe Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 8 Nov 2018 12:01:40 +0100 Subject: [PATCH 17/61] fix: [term_management] delete tags --- .../terms/templates/terms_management.html | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index b84b7042..a2ecd906 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -80,7 +80,7 @@
- {% set uniq_id = 0 %} + {% set uniq_id = namespace(modal_id=0)%}
@@ -213,9 +213,9 @@ {% endfor %} {% if notificationTagsTermMapping[regex] %} -
+
- - {% set uniq_id = uniq_id + 1 %} + {% set uniq_id.modal_id = uniq_id.modal_id + 1 %} {% endif %}
@@ -290,9 +290,9 @@ {% endfor %} {% if notificationTagsTermMapping[term] %} -
+
- - {% set uniq_id = uniq_id + 1 %} + {% set uniq_id.modal_id = uniq_id.modal_id + 1 %} {% endif %}
From c3903ab203a197bb6106fb88a3a11546f661ff68 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 9 Nov 2018 13:58:09 +0100 Subject: [PATCH 18/61] fix: [MISP_The_Hive_feeder] handle not saved pastes --- bin/MISP_The_Hive_feeder.py | 78 +++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index ff6552f2..07c121c9 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -90,6 +90,41 @@ def create_the_hive_alert(source, path, tag): except: print('hive connection error') +def feeder(message, count=0): + + if flag_the_hive or flag_misp: + tag, path = message.split(';') + ## FIXME: remove it + if PASTES_FOLDER not in path: + path = os.path.join(PASTES_FOLDER, path) + try: + paste = Paste.Paste(path) + except FileNotFoundError: + if count < 10: + r_serv_db.zincrby('mess_not_saved_export', message, 1) + return 0 + else: + r_serv_db.zrem('mess_not_saved_export', message) + print('Error: {} do not exist, tag= {}'.format(path, tag)) + return 0 + + source = '/'.join(paste.p_path.split('/')[-6:]) + + if HiveApi != False: + if int(r_serv_db.get('hive:auto-alerts')) == 1: + whitelist_hive = r_serv_db.scard('whitelist_hive') + if r_serv_db.sismember('whitelist_hive', tag): + create_the_hive_alert(source, path, tag) + else: + print('hive, auto alerts creation disable') + if flag_misp: + if int(r_serv_db.get('misp:auto-events')) == 1: + if r_serv_db.sismember('whitelist_misp', tag): + misp_wrapper.pushToMISP(uuid_ail, path, tag) + else: + print('misp, auto events creation disable') + + if __name__ == "__main__": publisher.port = 6380 @@ -174,37 +209,30 @@ if __name__ == "__main__": r_serv_db.set('ail:thehive', False) print('Not connected to The HIVE') + refresh_time = 3 ## FIXME: remove it PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + time_1 = time.time() while True: # Get one message from the input queue message = p.get_from_set() if message is None: - publisher.debug("{} queue is empty, waiting 1s".format(config_section)) - time.sleep(1) - continue + + # handle not saved pastes + if int(time.time() - time_1) > refresh_time: + + num_queu = r_serv_db.zcard('mess_not_saved_export') + list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True) + + if num_queu and list_queu: + for i in range(0, num_queu): + feeder(list_queu[i][0],list_queu[i][1]) + + time_1 = time.time() + else: + publisher.debug("{} queue is empty, waiting 1s".format(config_section)) + time.sleep(1) else: - - if flag_the_hive or flag_misp: - tag, path = message.split(';') - ## FIXME: remove it - if PASTES_FOLDER not in path: - path = os.path.join(PASTES_FOLDER, path) - paste = Paste.Paste(path) - source = '/'.join(paste.p_path.split('/')[-6:]) - - if HiveApi != False: - if int(r_serv_db.get('hive:auto-alerts')) == 1: - whitelist_hive = r_serv_db.scard('whitelist_hive') - if r_serv_db.sismember('whitelist_hive', tag): - create_the_hive_alert(source, path, tag) - else: - print('hive, auto alerts creation disable') - if flag_misp: - if int(r_serv_db.get('misp:auto-events')) == 1: - if r_serv_db.sismember('whitelist_misp', tag): - misp_wrapper.pushToMISP(uuid_ail, path, tag) - else: - print('misp, auto events creation disable') + feeder(message) From b31618eb6a5178c52b28bcbc00ed75c2de2833eb Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 9 Nov 2018 15:26:26 +0100 Subject: [PATCH 19/61] chg: [Mixer] config default feeder name #291 --- bin/Mixer.py | 3 ++- bin/packages/config.cfg.sample | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/Mixer.py b/bin/Mixer.py index e1656b8e..e7f9e6de 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -80,6 +80,7 @@ if __name__ == '__main__': # OTHER CONFIG # operation_mode = cfg.getint("Module_Mixer", "operation_mode") ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate") + default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name") # STATS # processed_paste = 0 @@ -106,7 +107,7 @@ if __name__ == '__main__': paste_name = complete_paste except ValueError as e: - feeder_name = "unnamed_feeder" + feeder_name = default_unnamed_feed_name paste_name = complete_paste # Processed paste diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 97cd692a..6efb4be4 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -99,6 +99,7 @@ threshold_stucked_module=600 operation_mode = 3 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) ttl_duplicate = 86400 +default_unnamed_feed_name = unnamed_feeder [RegexForTermsFrequency] max_execution_time = 60 From cb41c54135e8aa487a728de943ba243e9bea9043 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 21 Nov 2018 08:49:58 +0100 Subject: [PATCH 20/61] fix: [lib_refine] add dns configuration --- bin/packages/config.cfg.sample | 3 +++ bin/packages/lib_refine.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6efb4be4..b5980766 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -206,6 +206,9 @@ dns = 8.8.8.8 [Mail] dns = 8.8.8.8 +[Web] +dns = 149.13.33.69 + # Indexer configuration [Indexer] type = whoosh diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py index 5d2af0a9..32f56900 100644 --- a/bin/packages/lib_refine.py +++ b/bin/packages/lib_refine.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 import re +import os +import configparser import dns.resolver from pubsublogger import publisher @@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns): def checking_A_record(r_serv, domains_set): + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + dns_server = cfg.get("Web", "dns") + score = 0 num = len(domains_set) WalidA = set([]) resolver = dns.resolver.Resolver() - resolver.nameservers = ['149.13.33.69'] + resolver.nameservers = [dns_server] resolver.timeout = 5 resolver.lifetime = 2 From e4ced241ebbaa34ce638207c1271ca3d0c1d855d Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 14:57:24 +0900 Subject: [PATCH 21/61] chg: [python] Added necessary checks in LAUNCH.sh to be able to launch the script without doing bin/activate chg: [doc] Added correct pystemon path fix: [zmq] Small ZMQ test fix. fix: [doc] Moved the print statement in feeder so it is less verbose but still useful --- HOWTO.md | 2 +- README.md | 2 - bin/LAUNCH.sh | 109 +++++++++++++++++++--------------- bin/feeder/pystemon-feeder.py | 2 +- bin/feeder/test-zmq.py | 2 +- 5 files changed, 65 insertions(+), 52 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index 2228d3a6..c21a970f 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -27,7 +27,7 @@ Feed data to AIL: 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly -5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` +5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ``` How to create a new module diff --git a/README.md b/README.md index 72166c58..5e014852 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework ./installing_deps.sh -cd var/www/ -./update_thirdparty.sh cd ~/AIL-framework/ . ./AILENV/bin/activate cd bin/ diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..7b217f88 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -9,11 +9,26 @@ WHITE="\\033[0;02m" YELLOW="\\033[1;33m" CYAN="\\033[1;36m" -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; +# Getting CWD where bash script resides +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIL_HOME="${DIR}" + +cd ${AIL_HOME} + +if [ -e "${DIR}/AILENV/bin/python" ]; then + echo "AIL-framework virtualenv seems to exist, good" + ENV_PY="${DIR}/AILENV/bin/python" +else + echo "Please make sure you have a AIL-framework environment, au revoir" + exit 1 +fi + +[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 + +export AIL_BIN=${AIL_HOME}/bin/ +export AIL_FLASK=${AIL_HOME}/var/www/ +export AIL_REDIS=${AIL_HOME}/redis/src/ +export AIL_ARDB=${AIL_HOME}/ardb/src/ export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH @@ -101,16 +116,16 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x' } function checking_configuration { bin_dir=${AIL_HOME}/bin echo -e "\t* Checking configuration" if [ "$1" == "automatic" ]; then - bash -c "python3 $bin_dir/Update-conf.py True" + bash -c "${ENV_PY} $bin_dir/Update-conf.py True" else - bash -c "python3 $bin_dir/Update-conf.py False" + bash -c "${ENV_PY} $bin_dir/Update-conf.py False" fi exitStatus=$? @@ -128,75 +143,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x' } @@ -219,7 +234,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x' sleep 0.1 done @@ -342,7 +357,7 @@ function launch_flask { screen -dmS "Flask_AIL" sleep 0.1 echo -e $GREEN"\t* Launching Flask server"$DEFAULT - screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" + screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x" else echo -e $RED"\t* A Flask screen is already launched"$DEFAULT fi @@ -353,9 +368,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x' sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x' else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index b6680ee9..280849ba 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -61,10 +61,10 @@ topic = '102' while True: time.sleep(base_sleeptime + sleep_inc) paste = r.lpop("pastes") - print(paste) if paste is None: continue try: + print(paste) with open(pystemonpath+paste, 'rb') as f: #.read() messagedata = f.read() path_to_send = pastes_directory+paste diff --git a/bin/feeder/test-zmq.py b/bin/feeder/test-zmq.py index f6f28aa1..110c5de2 100644 --- a/bin/feeder/test-zmq.py +++ b/bin/feeder/test-zmq.py @@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port) # 102 Full pastes in raw base64(gz) topicfilter = "102" -socket.setsockopt(zmq.SUBSCRIBE, topicfilter) +socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter) while True: message = socket.recv() From 41a34b3519d1a3a7cc0e06da5f81d79a6d772e55 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:08:59 +0900 Subject: [PATCH 22/61] fix: [bash] As we instantiate from a subdirectory, we need to take this into account. --- bin/LAUNCH.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 7b217f88..c8571866 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -10,7 +10,7 @@ YELLOW="\\033[1;33m" CYAN="\\033[1;36m" # Getting CWD where bash script resides -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" AIL_HOME="${DIR}" cd ${AIL_HOME} From 6522cffd57a10b662da1351314e79cf1422bafbe Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:10:57 +0900 Subject: [PATCH 23/61] chg: [doc] Added note that redis-server is bundled during install --- bin/LAUNCH.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c8571866..abc49d92 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -23,7 +23,8 @@ else exit 1 fi -[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 +# redis-server is bundled during install +## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 export AIL_BIN=${AIL_HOME}/bin/ export AIL_FLASK=${AIL_HOME}/var/www/ From 33bc23a297a4288f4ab394da414536532a60b629 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:22:35 +0900 Subject: [PATCH 24/61] fix: [bash] evoking bash -c and splitting vars in this fashion wants to be changed in future revisions. --- bin/LAUNCH.sh | 78 +++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index abc49d92..b46e73fc 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -117,7 +117,7 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' launch_queues.py; read x' } function checking_configuration { @@ -144,75 +144,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModulesInformationV2.py -k 0 -c 1; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mixer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Global.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Duplicates.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Lines.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./DomClassifier.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Categ.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tokenize.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CreditCards.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./BankAccount.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Onion.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mail.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ApiKey.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Web.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Credential.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Curve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CurveManageTopSets.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./RegexForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SetForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Indexer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Keys.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Decoder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Bitcoin.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Phone.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Release.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Cve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./WebStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModuleStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./LibInjection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./alertHandler.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./MISP_The_Hive_feeder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tags.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SentimentAnalysis.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./submit_paste.py; read x' } @@ -235,7 +235,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Crawler.py onion '$i'; read x' sleep 0.1 done @@ -369,9 +369,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./feeder/pystemon-feeder.py; read x' sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; '${ENV_PY} ' ./pystemon.py; read x' else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi From 88e561aa20705a060cacaf0b3316e58fada0d281 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:41:45 +0900 Subject: [PATCH 25/61] chg: [bash] Replace '' with "" --- bin/LAUNCH.sh | 82 +++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b46e73fc..19a96cc0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -107,9 +107,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { @@ -117,7 +117,7 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x" } function checking_configuration { @@ -144,75 +144,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" } @@ -235,7 +235,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x" sleep 0.1 done @@ -369,9 +369,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x" sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; '${ENV_PY} ' ./pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x" else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi From f82c899e7bed6581e8048f672ab587dc14514c23 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:48:12 +0900 Subject: [PATCH 26/61] fix: [bash] fixed log_subscriber path --- bin/LAUNCH.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 19a96cc0..c4aba1cc 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -30,6 +30,7 @@ export AIL_BIN=${AIL_HOME}/bin/ export AIL_FLASK=${AIL_HOME}/var/www/ export AIL_REDIS=${AIL_HOME}/redis/src/ export AIL_ARDB=${AIL_HOME}/ardb/src/ +export AIL_VENV=${AIL_HOME}/AILENV/ export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH @@ -107,9 +108,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Queuing -l ../logs/; read x" + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Script -l ../logs/; read x" + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { From b60fd55efed595026ce892aced821415c7daed7c Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 16:16:31 +0900 Subject: [PATCH 27/61] chg: [bash] Small PATH precedence issue. --- bin/LAUNCH.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c4aba1cc..549c0425 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -11,7 +11,7 @@ CYAN="\\033[1;36m" # Getting CWD where bash script resides DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" -AIL_HOME="${DIR}" +export AIL_HOME="${DIR}" cd ${AIL_HOME} @@ -32,6 +32,7 @@ export AIL_REDIS=${AIL_HOME}/redis/src/ export AIL_ARDB=${AIL_HOME}/ardb/src/ export AIL_VENV=${AIL_HOME}/AILENV/ +export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH export PATH=$AIL_ARDB:$PATH From 9c0a65cad87819fa171d0f6f4313d43fe420f686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Thu, 29 Nov 2018 08:58:24 +0100 Subject: [PATCH 28/61] fix: [requirement] force redis version --- pip3_packages_requirement.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index dd447d5c..12ef09ff 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -2,7 +2,7 @@ pymisp thehive4py -redis +redis==2.10.6 #filemagic conflict with magic crcmod mmh3 From 423ddb6a3429fd623e9cc6f892fa15c147dc7b4b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 3 Dec 2018 17:34:11 +0100 Subject: [PATCH 29/61] training added --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9b447bac..fbcb5363 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,11 @@ Eventually you can browse the status of the AIL framework website at the followi http://localhost:7000/ ``` +Training +-------- + +CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg. + HOWTO ----- From 08661eda4e3c34c285d5d53ee95ffdf54cd1cc78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Wed, 5 Dec 2018 08:57:17 +0100 Subject: [PATCH 30/61] fix: [requirements] remove duplicate entry, #296 --- pip3_packages_requirement.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index 12ef09ff..3991e158 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -13,7 +13,6 @@ zmq langid #Essential -redis pyzmq dnspython logbook From 3ff991995e61689e15900abc3e944cc3e0146fb9 Mon Sep 17 00:00:00 2001 From: Mike Eriksson Date: Sun, 9 Dec 2018 12:29:31 +0000 Subject: [PATCH 31/61] Changed the Dockerfile so that it starts with installing any and all outstanding updates for Ubuntu --- Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 71318ba4..340e5014 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,19 @@ FROM ubuntu:16.04 -RUN mkdir /opt/AIL && apt-get update -y \ - && apt-get install git python-dev build-essential \ - libffi-dev libssl-dev libfuzzy-dev wget sudo -y +# Make sure that all updates are in place +RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ + && apt-get dist-upgrade -y && apt-get autoremove -y + +# Install needed packages +RUN apt-get install git python-dev build-essential \ + libffi-dev libssl-dev libfuzzy-dev wget sudo -y # Adding sudo command RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # Installing AIL dependencies +RUN mkdir /opt/AIL ADD . /opt/AIL WORKDIR /opt/AIL RUN ./installing_deps.sh From 3051c7280f234034f2e9791660edf9df389ed2ca Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 11 Dec 2018 16:55:47 +0100 Subject: [PATCH 32/61] fix: [install script] use virtualenv package --- installing_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installing_deps.sh b/installing_deps.sh index 6110d534..484ca770 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -5,7 +5,7 @@ set -x sudo apt-get update -sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \ +sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ screen g++ python-tk unzip libsnappy-dev cmake -y #optional tor install From 4e08aaa80f3d7364f67bdfd7c3dab4d1c2711a78 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 17 Dec 2018 15:58:48 +0100 Subject: [PATCH 33/61] chg: [UI Crawled Domains] show crawled domains by date range (options: domains tags, up domains, down domains) todo: Shows domain history --- bin/packages/HiddenServices.py | 8 +- .../hiddenServices/Flask_hiddenServices.py | 100 +++++- .../hiddenServices/templates/domains.html | 291 ++++++++++++++++++ .../templates/hiddenServices.html | 177 ++++++++--- 4 files changed, 530 insertions(+), 46 deletions(-) create mode 100644 var/www/modules/hiddenServices/templates/domains.html diff --git a/bin/packages/HiddenServices.py b/bin/packages/HiddenServices.py index d515c955..92e2e57c 100755 --- a/bin/packages/HiddenServices.py +++ b/bin/packages/HiddenServices.py @@ -81,8 +81,12 @@ class HiddenServices(object): return '' return origin_paste.replace(self.paste_directory+'/', '') - def get_domain_tags(self): - return self.tags + def get_domain_tags(self, update=False): + if not update: + return self.tags + else: + self.get_last_crawled_pastes() + return self.tags def update_domain_tags(self, children): p_tags = self.r_serv_metadata.smembers('tag:'+children) diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 47ea56f1..4421505d 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -8,7 +8,7 @@ import redis import datetime import sys import os -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for from Date import Date from HiddenServices import HiddenServices @@ -40,6 +40,16 @@ def get_date_range(num_day): return list(reversed(date_list)) +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + def unpack_paste_tags(p_tags): l_tags = [] for tag in p_tags: @@ -94,6 +104,94 @@ def hiddenServices_page(): return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains) +@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST']) +def get_onions_by_daterange(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + domains_up = request.form.get('domains_up') + domains_down = request.form.get('domains_down') + domains_tags = request.form.get('domains_tags') + + return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags)) + +@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET']) +def show_domains_by_daterange(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + domains_up = request.args.get('domains_up') + domains_down = request.args.get('domains_down') + domains_tags = request.args.get('domains_tags') + + date_range = [] + if date_from is not None and date_to is not None: + #change format + try: + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + except: + pass + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] + date_to = date_from + + else: + date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] + date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] + + domains_by_day = {} + domain_metadata = {} + for date in date_range: + if domains_up: + domains_up = True + domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date))) + for domain in domains_by_day[date]: + h = HiddenServices(domain, 'onion') + domain_metadata[domain] = {} + if domains_tags: + domains_tags = True + domain_metadata[domain]['tags'] = h.get_domain_tags(update=True) + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + domain_metadata[domain]['status_text'] = 'UP' + domain_metadata[domain]['status_color'] = 'Green' + domain_metadata[domain]['status_icon'] = 'fa-check-circle' + + if domains_down: + domains_down = True + domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date))) + if domains_up: + domains_by_day[date].extend(domains_by_day_down) + else: + domains_by_day[date] = domains_by_day_down + for domain in domains_by_day_down: + #h = HiddenServices(onion_domain, 'onion') + domain_metadata[domain] = {} + #domain_metadata[domain]['tags'] = h.get_domain_tags() + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + + domain_metadata[domain]['status_text'] = 'DOWN' + domain_metadata[domain]['status_color'] = 'Red' + domain_metadata[domain]['status_icon'] = 'fa-times-circle' + + return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata, + date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, + domains_tags=domains_tags, bootstrap_label=bootstrap_label) + @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET']) def onion_domain(): onion_domain = request.args.get('onion_domain') diff --git a/var/www/modules/hiddenServices/templates/domains.html b/var/www/modules/hiddenServices/templates/domains.html new file mode 100644 index 00000000..136291b1 --- /dev/null +++ b/var/www/modules/hiddenServices/templates/domains.html @@ -0,0 +1,291 @@ + + + + + + + + Hidden Service - AIL + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+ +
+
+ + {% for date in date_range %} + {% if domains_by_day[date]%} +
+
+

{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}

+
+
+ + + + + + + + + + + + {% for domain in domains_by_day[date] %} + + + + + + + {% endfor %} + +
DomainFirst SeenLast CheckStatus
+ {{ domain }} +
+ {% for tag in domain_metadata[domain]['tags'] %} + + {{ tag }} {{ domain_metadata[domain]['tags'][tag] }} + + {% endfor %} +
+ 		{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}
+ + {{domain_metadata[domain]['status_text']}} +
+
+
+
+
+ {% endif %} + {% endfor %} + + +
+ +
+
+
+ Select domains by date range : +
+
+
+
+
+
+ + +
+
+ + +
+
+ +
+
+
+
+ +
+
+ +
+
+
+ + +
+
+ +
+ +
+ +
+ + + + + + + + diff --git a/var/www/modules/hiddenServices/templates/hiddenServices.html b/var/www/modules/hiddenServices/templates/hiddenServices.html index 59aeb2ae..b781a26a 100644 --- a/var/www/modules/hiddenServices/templates/hiddenServices.html +++ b/var/www/modules/hiddenServices/templates/hiddenServices.html @@ -12,11 +12,17 @@ + + +