From 78c611fead78cccc98f662150299e2eb5b535ee6 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 8 Aug 2016 09:17:44 +0200 Subject: [PATCH] Added warning_paste module and created related webpages. Fixed a Faup bug in credential (multiple instanciation) and added correc populate_set_out in concerned modules (creditcard, credential, ...). Linked browse_warning_paste module and Flask function with redis (created new sets). --- bin/Browse_warning_paste.py | 57 ++++++ bin/Credential.py | 5 +- bin/CreditCard.py | 2 + bin/Cve.py | 7 +- bin/Duplicate_ssdeep_v2.py | 2 +- bin/Keys.py | 5 +- bin/LAUNCH.sh | 2 + bin/Mail.py | 1 + bin/Phone.py | 4 +- bin/SQLInjectionDetection.py | 4 + bin/Url.py | 7 +- bin/packages/modules.cfg | 19 +- var/www/Flask_server.py | 22 ++- var/www/templates/Moduletrending.html | 1 + var/www/templates/Trending.html | 1 + var/www/templates/browse_important_paste.html | 164 ++++++------------ .../templates/important_paste_by_module.html | 112 ++++++++++++ 17 files changed, 280 insertions(+), 135 deletions(-) create mode 100755 bin/Browse_warning_paste.py create mode 100644 var/www/templates/important_paste_by_module.html diff --git a/bin/Browse_warning_paste.py b/bin/Browse_warning_paste.py new file mode 100755 index 00000000..49444979 --- /dev/null +++ b/bin/Browse_warning_paste.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +""" +The Browse_warning_paste module +==================== + +This module saved signaled paste (logged as 'warning') in redis for further usage +like browsing by category + +Its input comes from other modules, namely: + Credential, CreditCard, SQLinjection, CVE, Keys, Mail and Phone + +""" + +import redis +import time +from datetime import datetime, timedelta +from packages import Paste +from pubsublogger import publisher +from Helper import Process + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'Browse_warning_paste' + + p = Process(config_section) + + server = redis.StrictRedis( + host=p.config.get("Redis_Level_DB", "host"), + port=p.config.get("Redis_Level_DB", "port"), + db=p.config.get("Redis_Level_DB", "db")) + + # FUNCTIONS # + publisher.info("Script duplicate started") + + while True: + message = p.get_from_set() + if message is not None: + module_name, p_path = message.split(';') + #PST = Paste.Paste(p_path) + else: + publisher.debug("Script Attribute is idling 10s") + time.sleep(10) + continue + + # Add in redis + # Format in set: WARNING_moduleName -> p_path + key = "WARNING_" + module_name + print key + ' -> ' + p_path + server.sadd(key, p_path) + + publisher.info('Saved in warning paste {}'.format(p_path)) + #print 'Saved in warning paste {}'.format(p_path) + diff --git a/bin/Credential.py b/bin/Credential.py index 23c90506..cfbec7c7 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -15,6 +15,8 @@ if __name__ == "__main__": p = Process(config_section) publisher.info("Find credentials") + faup = Faup() + critical = 8 regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" @@ -55,10 +57,11 @@ if __name__ == "__main__": publisher.warning(to_print) #Send to duplicate p.populate_set_out(filepath, 'Duplicate') + #send to Browse_warning_paste + p.populate_set_out('credential;{}'.format(filepath), 'BrowseWarningPaste') #Put in form, count occurences, then send to moduleStats creds_sites = {} - faup = Faup() for url in sites: faup.decode(url) domain = faup.get()['domain'] diff --git a/bin/CreditCard.py b/bin/CreditCard.py index de90f4d4..e5f8020e 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCard.py @@ -67,6 +67,8 @@ if __name__ == "__main__": to_print, len(creditcard_set))) #Send to duplicate p.populate_set_out(filepath, 'Redis_Duplicate') + #send to Browse_warning_paste + p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste') else: publisher.info('{}CreditCard related'.format(to_print)) else: diff --git a/bin/Cve.py b/bin/Cve.py index 7323ee5a..1e152463 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -53,5 +53,8 @@ if __name__ == '__main__': # Do something with the message from the queue search_cve(message) - # (Optional) Send that thing to the next queue - #p.populate_set_out(something_has_been_done) + #send to Browse_warning_paste + filepath, count = message.split() + p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') + #Send to duplicate + p.populate_set_out(filepath, 'Duplicate') diff --git a/bin/Duplicate_ssdeep_v2.py b/bin/Duplicate_ssdeep_v2.py index 22498b90..a2ab55aa 100755 --- a/bin/Duplicate_ssdeep_v2.py +++ b/bin/Duplicate_ssdeep_v2.py @@ -7,7 +7,7 @@ The Duplicate module This huge module is, in short term, checking duplicates. Its input comes from other modules, namely: - Credential, CreditCard, Keys, Mails and Phone + Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone This one differ from v1 by only using redis and not json file stored on disk diff --git a/bin/Keys.py b/bin/Keys.py index 9c44f60a..a286dada 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -17,7 +17,9 @@ def search_gpg(message): if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) #Send to duplicate - p.populate_set_out(message) + p.populate_set_out(message, 'Duplicate') + #send to Browse_warning_paste + p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste') if __name__ == '__main__': @@ -49,4 +51,3 @@ if __name__ == '__main__': search_gpg(message) # (Optional) Send that thing to the next queue - #p.populate_set_out(something_has_been_done) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b74dc455..86877c2a 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -144,6 +144,8 @@ function launching_scripts { screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' } #If no params, display the help diff --git a/bin/Mail.py b/bin/Mail.py index d3968442..2b3ed5fc 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -69,6 +69,7 @@ if __name__ == "__main__": for mail in MX_values[1]: print 'mail;{};{};{}'.format(1, mail, PST.p_date) p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats') + p.populate_set_out('mail;{}'.format(filename), 'BrowseWarningPaste') prec_filename = filename diff --git a/bin/Phone.py b/bin/Phone.py index b53b079c..b25dae41 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -23,8 +23,10 @@ def search_phone(message): if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) + #send to Browse_warning_paste + p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate - p.populate_set_out(message) + p.populate_set_out(message, 'Duplicate') if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index 9dae63bd..d6c3efa5 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -74,6 +74,10 @@ def analyse(url, path): print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL") publisher.warning(to_print) + #Send to duplicate + p.populate_set_out(path, 'Duplicate') + #send to Browse_warning_paste + p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) diff --git a/bin/Url.py b/bin/Url.py index b01c2725..398ca49d 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -103,10 +103,11 @@ if __name__ == "__main__": print hostl, asn, cc, \ pycountry.countries.get(alpha2=cc).name if cc == cc_critical: - publisher.warning( - 'Url;{};{};{};Detected {} {}'.format( + to_print = 'Url;{};{};{};Detected {} {}'.format( PST.p_source, PST.p_date, PST.p_name, - hostl, cc)) + hostl, cc) + #publisher.warning(to_print) + print to_print else: print hostl, asn, cc diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 56a2f6be..546ddef7 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -31,11 +31,11 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re [CreditCards] subscribe = Redis_CreditCard -publish = Redis_Duplicate,Redis_ModuleStats +publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste [Mail] subscribe = Redis_Mail -publish = Redis_Duplicate,Redis_ModuleStats +publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste [Onion] subscribe = Redis_Onion @@ -54,27 +54,36 @@ subscribe = Redis_Url [SQLInjectionDetection] subscribe = Redis_Url +publish = Redis_BrowseWarningPaste,Redis_Duplicate [ModuleStats] subscribe = Redis_ModuleStats +[Browse_warning_paste] +subscribe = Redis_BrowseWarningPaste + +#[send_to_queue] +#subscribe = Redis_Cve +#publish = Redis_BrowseWarningPaste + [Release] subscribe = Redis_Global [Credential] subscribe = Redis_Credential -publish = Redis_Duplicate,Redis_ModuleStats +publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste [Cve] subscribe = Redis_Cve +publish = Redis_Browse_warning_paste,Redis_Duplicate [Phone] subscribe = Redis_Global -publish = Redis_Duplicate +publish = Redis_Duplicate,Redis_BrowseWarningPaste [SourceCode] subscribe = Redis_SourceCode [Keys] subscribe = Redis_Global -publish = Redis_Duplicate +publish = Redis_Duplicate,Redis_BrowseWarningPaste diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 33f85c0e..ee2735b3 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -44,6 +44,10 @@ r_serv_charts = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_Trending", "port"), db=cfg.getint("Redis_Level_DB_Trending", "db")) +r_serv_db = redis.StrictRedis( + host=cfg.get("Redis_Level_DB", "host"), + port=cfg.getint("Redis_Level_DB", "port"), + db=cfg.getint("Redis_Level_DB", "db")) app = Flask(__name__, static_url_path='/static/') @@ -157,9 +161,11 @@ def showpaste(content_range): return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list) -def getPastebyType(module_name): +def getPastebyType(server, module_name): all_path = [] - all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz") + for path in server.smembers('WARNING_'+module_name): + #all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz") + all_path.append(path) return all_path @@ -377,13 +383,19 @@ def trending(): @app.route("/browseImportantPaste/", methods=['GET']) def browseImportantPaste(): module_name = request.args.get('moduleName') + return render_template("browse_important_paste.html") + + +@app.route("/importantPasteByModule/", methods=['GET']) +def importantPasteByModule(): + module_name = request.args.get('moduleName') all_content = [] paste_date = [] paste_linenum = [] all_path = [] - for path in getPastebyType(module_name): + for path in getPastebyType(r_serv_db, module_name): all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') @@ -394,9 +406,7 @@ def browseImportantPaste(): paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) - return render_template("browse_important_paste.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal) - - + return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal) @app.route("/moduletrending/") def moduletrending(): diff --git a/var/www/templates/Moduletrending.html b/var/www/templates/Moduletrending.html index d3ddc31b..0e51f95e 100644 --- a/var/www/templates/Moduletrending.html +++ b/var/www/templates/Moduletrending.html @@ -29,6 +29,7 @@
  • Dashboard
  • Trending charts
  • Modules statistics
    • +
  • Browse important pastes
    • diff --git a/var/www/templates/Trending.html b/var/www/templates/Trending.html index 26493a87..f27cad84 100644 --- a/var/www/templates/Trending.html +++ b/var/www/templates/Trending.html @@ -33,6 +33,7 @@
  • Dashboard
  • Trending charts
  • Modules statistics
    • +
  • Browse important pastes
    • diff --git a/var/www/templates/browse_important_paste.html b/var/www/templates/browse_important_paste.html index d9c2de8b..881096f4 100644 --- a/var/www/templates/browse_important_paste.html +++ b/var/www/templates/browse_important_paste.html @@ -16,7 +16,7 @@ - +