diff --git a/bin/Categ.py b/bin/Categ.py index 74d568d7..9449e87c 100755 --- a/bin/Categ.py +++ b/bin/Categ.py @@ -66,7 +66,7 @@ if __name__ == "__main__": # FUNCTIONS # publisher.info("Script Categ started") - categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential'] + categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve'] tmp_dict = {} for filename in categories: bname = os.path.basename(filename) diff --git a/bin/Cve.py b/bin/Cve.py new file mode 100755 index 00000000..7323ee5a --- /dev/null +++ b/bin/Cve.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules +""" + +import time +import re +from pubsublogger import publisher +from packages import Paste +from Helper import Process + + +def search_cve(message): + filepath, count = message.split() + paste = Paste.Paste(filepath) + content = paste.get_p_content() + # regex to find CVE + reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') + # list of the regex results in the Paste, may be null + results = set(reg_cve.findall(content)) + + # if the list is greater than 2, we consider the Paste may contain a list of cve + if len(results) > 0: + print('{} contains CVEs'.format(paste.p_name)) + publisher.warning('{} contains CVEs'.format(paste.p_name)) + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'Cve' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Run CVE module") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + search_cve(message) + + # (Optional) Send that thing to the next queue + #p.populate_set_out(something_has_been_done) diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index 0712e959..14a417f2 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -39,7 +39,7 @@ def main(): if message is not None: PST = Paste.Paste(message) else: - publisher.debug("Script DomClassifier is idling 10s") + publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() diff --git a/bin/Duplicate.py b/bin/Duplicate.py index ed62be0d..a7a41dc1 100755 --- a/bin/Duplicate.py +++ b/bin/Duplicate.py @@ -33,11 +33,12 @@ if __name__ == "__main__": # DB OBJECT & HASHS ( DISK ) # FIXME increase flexibility dico_redis = {} - for year in xrange(2013, 2015): + for year in xrange(2013, 2017): for month in xrange(0, 16): dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), port=year, db=month) + #print("dup: "+str(year)+str(month).zfill(2)+"\n") # FUNCTIONS # publisher.info("Script duplicate started") @@ -81,7 +82,7 @@ if __name__ == "__main__": bloop_path_set.add(filebloompath) # UNIQUE INDEX HASHS TABLE - r_serv0 = dico_redis["201300"] + r_serv0 = dico_redis["201600"] r_serv0.incr("current_index") index = r_serv0.get("current_index")+str(PST.p_date) # HASHTABLES PER MONTH (because of r_serv1 changing db) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index d033273a..fc8c9ff1 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,6 +66,7 @@ function launching_lvldb { lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" db1_y='2013' db2_y='2014' + db3_y='2016' nb_db=13 screen -dmS "LevelDB" @@ -75,6 +76,8 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' + sleep 0.1 + screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' } function launching_logs { @@ -127,6 +130,16 @@ function launching_scripts { screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' } #If no params, display the help diff --git a/bin/Url.py b/bin/Url.py index 472d73b3..0738d1ce 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -7,6 +7,8 @@ import dns.exception from packages import Paste from packages import lib_refine from pubsublogger import publisher +from pyfaup.faup import Faup +import re # Country and ASN lookup from cymru.ip2asn.dns import DNSClient as ip2asn @@ -16,6 +18,13 @@ import ipaddress from Helper import Process +# Used to prevent concat with empty fields due to url parsing +def avoidNone(str): + if str is None: + return "" + else: + return str + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" @@ -41,6 +50,7 @@ if __name__ == "__main__": message = p.get_from_set() prec_filename = None + faup = Faup() url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" @@ -53,19 +63,27 @@ if __name__ == "__main__": PST = Paste.Paste(filename) client = ip2asn() for x in PST.get_regex(url_regex): - scheme, credential, subdomain, domain, host, tld, \ - port, resource_path, query_string, f1, f2, f3, \ - f4 = x + matching_url = re.search(url_regex, PST.get_p_content()) + url = matching_url.group(0) + + to_send = "{} {}".format(url, PST._get_p_date()) + p.populate_set_out(to_send, 'Url') + + faup.decode(url) + domain = faup.get_domain() + subdomain = faup.get_subdomain() + f1 = None + domains_list.append(domain) - p.populate_set_out(x, 'Url') - publisher.debug('{} Published'.format(x)) + + publisher.debug('{} Published'.format(url)) if f1 == "onion": print domain - hostl = unicode(subdomain+domain) + hostl = unicode(avoidNone(subdomain)+avoidNone(domain)) try: - socket.setdefaulttimeout(2) + socket.setdefaulttimeout(1) ip = socket.gethostbyname(unicode(hostl)) except: # If the resolver is not giving any IPv4 address, diff --git a/bin/WebStats.py b/bin/WebStats.py new file mode 100755 index 00000000..5da443a8 --- /dev/null +++ b/bin/WebStats.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules +""" + +import time +import datetime +import re +import redis +import os +from packages import lib_words +from pubsublogger import publisher +from packages import Paste +from Helper import Process +from pyfaup.faup import Faup + +def analyse(field_name): + field = url_parsed[field_name] + if field is not None: + prev_score = r_serv1.hget(field, date) + if prev_score is not None: + r_serv1.hset(field, date, int(prev_score) + 1) + else: + r_serv1.hset(field, date, 1) + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'WebStats' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Makes statistics about valid URL") + + # REDIS # + r_serv1 = redis.StrictRedis( + host=p.config.get("Redis_Level_DB", "host"), + port=p.config.get("Redis_Level_DB", "port"), + db=p.config.get("Redis_Level_DB", "db")) + + # FILE CURVE SECTION # + csv_path_proto = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolstrending_csv")) + protocolsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolsfile")) + + csv_path_tld = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "tldstrending_csv")) + tldsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "tldsfile")) + + faup = Faup() + generate_new_graph = False + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + if generate_new_graph: + generate_new_graph = False + print 'Building graph' + today = datetime.date.today() + year = today.year + month = today.month + + lib_words.create_curve_with_word_file(r_serv1, csv_path_proto, + protocolsfile_path, year, + month) + + lib_words.create_curve_with_word_file(r_serv1, csv_path_tld, + tldsfile_path, year, + month) + + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + else: + generate_new_graph = True + # Do something with the message from the queue + url, date = message.split() + faup.decode(url) + url_parsed = faup.get() + + analyse('scheme') #Scheme analysis + analyse('tld') #Tld analysis diff --git a/bin/empty_queue.py b/bin/empty_queue.py new file mode 100755 index 00000000..a5ccae68 --- /dev/null +++ b/bin/empty_queue.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +""" +The Empty queue module +==================== + +This simple module can be used to clean all queues. + +Requirements: +------------- + + +""" +import redis +import os +import time +from packages import Paste +from pubsublogger import publisher +from Helper import Process + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys'] + + for queue in config_section: + print 'dropping: ' + queue + p = Process(queue) + while True: + message = p.get_from_set() + if message is None: + break + diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 94a2fee4..9d8d6637 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -27,7 +27,7 @@ subscribe = Redis_Words [Categ] subscribe = Redis_Global -publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode +publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve [CreditCards] subscribe = Redis_CreditCards @@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion subscribe = Redis_Web publish = Redis_Url,ZMQ_Url +[WebStats] +subscribe = Redis_Url + [Release] subscribe = Redis_Global [Credential] subscribe = Redis_Credential +[Cve] +subscribe = Redis_Cve + [Phone] subscribe = Redis_Global diff --git a/files/Cve b/files/Cve new file mode 100644 index 00000000..1d7b65c5 --- /dev/null +++ b/files/Cve @@ -0,0 +1 @@ +CVE diff --git a/files/protocolsfile b/files/protocolsfile new file mode 100644 index 00000000..ffece4b6 --- /dev/null +++ b/files/protocolsfile @@ -0,0 +1,100 @@ +afs +file +ftp +z39.50 +z39.50r +z39.50s +vemmi +urn +nfs +dict +acap +rtspu +rtsp +rtsps +tip +pop +cid +mid +data +thismessage +service +shttp +fax +modem +tv +sip +sips +go +icap +h323 +ipp +xmlrpc.beep +xmlrpc.beeps +tftp +mupdate +pres +im +mtqp +tel +iris +iris.beep +crid +snmp +tag +wais +prospero +soap.beep +soap.beeps +telnet +gopher +cap +info +dns +ldap +dav +opaquelocktoken +msrp +msrps +dtn +imap +xmpp +iax +news +nntp +snews +sms +rsync +sieve +geo +mailto +jms +mailserver +ipn +tn3270 +ws +wss +xcon +xcon-userid +about +aaa +aaas +session +ni +nih +reload +ham +stun +stuns +turn +turns +http +https +coap +coaps +rtmfp +ipps +pkcs11 +acct +example +vnc diff --git a/files/tldsfile b/files/tldsfile new file mode 100644 index 00000000..73bff3af --- /dev/null +++ b/files/tldsfile @@ -0,0 +1,263 @@ +com +org +net +int +edu +gov +mil +arpa +ac +ad +ae +af +ag +ai +al +am +an +ao +aq +ar +as +at +au +aw +ax +az +ba +bb +bd +be +bf +bg +bh +bi +bj +bl +bm +bn +bo +bq +br +bs +bt +bv +bw +by +bz +ca +cc +cd +cf +cg +ch +ci +ck +cl +cm +cn +co +cr +cu +cv +cw +cx +cy +cz +de +dj +dk +dm +do +dz +ec +ee +eg +eh +er +es +et +eu +fi +fj +fk +fm +fo +fr +ga +gb +gd +ge +gf +gg +gh +gi +gl +gm +gn +gp +gq +gr +gs +gt +gu +gw +gy +hk +hm +hn +hr +ht +hu +id +ie +il +im +in +io +iq +ir +is +it +je +jm +jo +jp +ke +kg +kh +ki +km +kn +kp +kr +kw +ky +kz +la +lb +lc +li +lk +lr +ls +lt +lu +lv +ly +ma +mc +md +me +mf +mg +mh +mk +ml +mm +mn +mo +mp +mq +mr +ms +mt +mu +mv +mw +mx +my +mz +na +nc +ne +nf +ng +ni +nl +no +np +nr +nu +nz +om +pa +pe +pf +pg +ph +pk +pl +pm +pn +pr +ps +pt +pw +py +qa +re +ro +rs +ru +rw +sa +sb +sc +sd +se +sg +sh +si +sj +sk +sl +sm +sn +so +sr +ss +st +su +sv +sx +sy +sz +tc +td +tf +tg +th +tj +tk +tl +tm +tn +to +tp +tr +tt +tv +tw +tz +ua +ug +uk +um +us +uy +uz +va +vc +ve +vg +vi +vn +vu +wf +ws +ye +yt +za +zm +zw diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 7a1aa472..9c152cab 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -96,5 +96,14 @@ def wordstrending(): return render_template("Wordstrending.html") +@app.route("/protocolstrending/") +def protocolstrending(): + return render_template("Protocolstrending.html") + +@app.route("/tldstrending/") +def tldstrending(): + return render_template("Tldstrending.html") + + if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/var/www/templates/Protocolstrending.html b/var/www/templates/Protocolstrending.html new file mode 100644 index 00000000..3dd26200 --- /dev/null +++ b/var/www/templates/Protocolstrending.html @@ -0,0 +1,196 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + +
+ +
+
+
+

ProtocolsTrendings

+
+ +
+ +
+
+
+
+ Protocols Trend +
+
+ + +
+
+
+ +
+ +
+
+ +
+
+ +
+ + +
+ + + + diff --git a/var/www/templates/Tldstrending.html b/var/www/templates/Tldstrending.html new file mode 100644 index 00000000..e2706f21 --- /dev/null +++ b/var/www/templates/Tldstrending.html @@ -0,0 +1,196 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + +
+ +
+
+
+

Top Level Domain Trending

+
+ +
+ +
+
+
+
+ Top Level Domain Trending +
+
+ + +
+
+
+ +
+ +
+
+ +
+
+ +
+ + +
+ + + + diff --git a/var/www/templates/Wordstrending.html b/var/www/templates/Wordstrending.html index eee8bbe8..d00376dc 100644 --- a/var/www/templates/Wordstrending.html +++ b/var/www/templates/Wordstrending.html @@ -24,7 +24,7 @@