From 3dc356dc5e04abfba56eb21c4183a02c6fa37501 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 30 Jun 2016 14:36:47 +0200 Subject: [PATCH 1/4] Getting Started: Initial configuration working on the laptop --- bin/Categ.py | 2 +- bin/DomClassifier.py | 2 +- bin/Duplicate.py | 5 +++-- bin/LAUNCH.sh | 17 +++++++++++++++-- bin/Url.py | 16 +++++++++++++++- bin/packages/modules.cfg | 8 +++++++- var/www/update_thirdparty.sh | 2 +- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/bin/Categ.py b/bin/Categ.py index 74d568d7..9449e87c 100755 --- a/bin/Categ.py +++ b/bin/Categ.py @@ -66,7 +66,7 @@ if __name__ == "__main__": # FUNCTIONS # publisher.info("Script Categ started") - categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential'] + categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve'] tmp_dict = {} for filename in categories: bname = os.path.basename(filename) diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index 0712e959..14a417f2 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -39,7 +39,7 @@ def main(): if message is not None: PST = Paste.Paste(message) else: - publisher.debug("Script DomClassifier is idling 10s") + publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() diff --git a/bin/Duplicate.py b/bin/Duplicate.py index ed62be0d..a7a41dc1 100755 --- a/bin/Duplicate.py +++ b/bin/Duplicate.py @@ -33,11 +33,12 @@ if __name__ == "__main__": # DB OBJECT & HASHS ( DISK ) # FIXME increase flexibility dico_redis = {} - for year in xrange(2013, 2015): + for year in xrange(2013, 2017): for month in xrange(0, 16): dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), port=year, db=month) + #print("dup: "+str(year)+str(month).zfill(2)+"\n") # FUNCTIONS # publisher.info("Script duplicate started") @@ -81,7 +82,7 @@ if __name__ == "__main__": bloop_path_set.add(filebloompath) # UNIQUE INDEX HASHS TABLE - r_serv0 = dico_redis["201300"] + r_serv0 = dico_redis["201600"] r_serv0.incr("current_index") index = r_serv0.get("current_index")+str(PST.p_date) # HASHTABLES PER MONTH (because of r_serv1 changing db) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index d033273a..330a23ce 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,6 +66,7 @@ function launching_lvldb { lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" db1_y='2013' db2_y='2014' + db3_y='2016' nb_db=13 screen -dmS "LevelDB" @@ -75,6 +76,8 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' + sleep 0.1 + screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' } function launching_logs { @@ -104,11 +107,11 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' + #screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' + #screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' sleep 0.1 @@ -127,6 +130,16 @@ function launching_scripts { screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' + sleep 0.1 + #screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' } #If no params, display the help diff --git a/bin/Url.py b/bin/Url.py index 472d73b3..d33e7e9f 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -57,7 +57,21 @@ if __name__ == "__main__": port, resource_path, query_string, f1, f2, f3, \ f4 = x domains_list.append(domain) - p.populate_set_out(x, 'Url') +# p.populate_set_out(x, 'Url') + temp_x = () + for i in range(0,13): + if x[i] == '': + temp_x += ('None', ) + else: + temp_x += (x[i], ) + temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \ + temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \ + temp_f4 = temp_x + + to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \ + temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\ + temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date()) + p.populate_set_out(to_send , 'Url') publisher.debug('{} Published'.format(x)) if f1 == "onion": diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 94a2fee4..9d8d6637 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -27,7 +27,7 @@ subscribe = Redis_Words [Categ] subscribe = Redis_Global -publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode +publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve [CreditCards] subscribe = Redis_CreditCards @@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion subscribe = Redis_Web publish = Redis_Url,ZMQ_Url +[WebStats] +subscribe = Redis_Url + [Release] subscribe = Redis_Global [Credential] subscribe = Redis_Credential +[Cve] +subscribe = Redis_Cve + [Phone] subscribe = Redis_Global diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 2b5f7703..b666a5d5 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2 JQVERSION="1.11.1" wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js -wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js +#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js #Here to fix an error about an hard dependency in a obscur script of bootstrap.. wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js From beeeb76de93f97ccf86e9f5546a6e3070b2bcfd0 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 30 Jun 2016 14:38:28 +0200 Subject: [PATCH 2/4] Added new modules and started WebTrending web interface --- bin/Cve.py | 57 +++++++ bin/WebStats.py | 95 +++++++++++ bin/empty_queue.py | 35 ++++ files/Cve | 1 + files/protocolsfile | 3 + var/www/templates/Protocolstrending.html | 196 +++++++++++++++++++++++ 6 files changed, 387 insertions(+) create mode 100755 bin/Cve.py create mode 100755 bin/WebStats.py create mode 100755 bin/empty_queue.py create mode 100644 files/Cve create mode 100644 files/protocolsfile create mode 100644 var/www/templates/Protocolstrending.html diff --git a/bin/Cve.py b/bin/Cve.py new file mode 100755 index 00000000..7323ee5a --- /dev/null +++ b/bin/Cve.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules +""" + +import time +import re +from pubsublogger import publisher +from packages import Paste +from Helper import Process + + +def search_cve(message): + filepath, count = message.split() + paste = Paste.Paste(filepath) + content = paste.get_p_content() + # regex to find CVE + reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') + # list of the regex results in the Paste, may be null + results = set(reg_cve.findall(content)) + + # if the list is greater than 2, we consider the Paste may contain a list of cve + if len(results) > 0: + print('{} contains CVEs'.format(paste.p_name)) + publisher.warning('{} contains CVEs'.format(paste.p_name)) + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'Cve' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Run CVE module") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + search_cve(message) + + # (Optional) Send that thing to the next queue + #p.populate_set_out(something_has_been_done) diff --git a/bin/WebStats.py b/bin/WebStats.py new file mode 100755 index 00000000..15508e52 --- /dev/null +++ b/bin/WebStats.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules +""" + +import time +import re +import redis +import os +from pubsublogger import publisher +from packages import Paste +from Helper import Process + + + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'WebStats' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Makes statistics about valid URL") + + # REDIS # + r_serv1 = redis.StrictRedis( + host=p.config.get("Redis_Level_DB", "host"), + port=p.config.get("Redis_Level_DB", "port"), + db=p.config.get("Redis_Level_DB", "db")) + + # FILE CURVE SECTION # + csv_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolstrending_csv")) + protocolsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolsfile")) + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + generate_new_graph = False + + if message is None: + if generate_new_graph: + generate_new_graph = False + print 'Building graph' + today = datetime.date.today() + year = today.year + month = today.month + lib_words.create_curve_with_word_file(r_serv1, csv_path, + protocolsfile_path, year, + month) + + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + else: + generate_new_graph = True + # Do something with the message from the queue + scheme, credential, subdomain, domain, host, tld, \ + port, resource_path, query_string, f1, f2, f3, \ + f4 , date= message.split() + + prev_score = r_serv1.hget(scheme, date) + if prev_score is not None: + r_serv1.hset(scheme, date, int(prev_score) + int(score)) + else: + r_serv1.hset(scheme, date, score) + + + + + + + + + + + + + + + + + + diff --git a/bin/empty_queue.py b/bin/empty_queue.py new file mode 100755 index 00000000..6f82af90 --- /dev/null +++ b/bin/empty_queue.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +""" +The Duplicate module +==================== + +This huge module is, in short term, checking duplicates. + +Requirements: +------------- + + +""" +import redis +import os +import time +from packages import Paste +from pubsublogger import publisher +from Helper import Process + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys'] + + for queue in config_section: + print 'dropping: ' + queue + p = Process(queue) + while True: + message = p.get_from_set() + if message is None: + break + diff --git a/files/Cve b/files/Cve new file mode 100644 index 00000000..1d7b65c5 --- /dev/null +++ b/files/Cve @@ -0,0 +1 @@ +CVE diff --git a/files/protocolsfile b/files/protocolsfile new file mode 100644 index 00000000..f36a40f5 --- /dev/null +++ b/files/protocolsfile @@ -0,0 +1,3 @@ +FTP +HTTP +HTTPS diff --git a/var/www/templates/Protocolstrending.html b/var/www/templates/Protocolstrending.html new file mode 100644 index 00000000..795ca0ab --- /dev/null +++ b/var/www/templates/Protocolstrending.html @@ -0,0 +1,196 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + +
+ +
+
+
+

ProtocolsTrendings

+
+ +
+ +
+
+
+
+ Protocols Trend +
+
+ + +
+
+
+ +
+ +
+
+ +
+
+ +
+ + +
+ + + + From 4b3101b7b6423237c339e3b4f0676b7504febd5d Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 1 Jul 2016 16:59:08 +0200 Subject: [PATCH 3/4] Added template tld. Modified URL using Faup and refactored WebStats. --- bin/LAUNCH.sh | 6 +- bin/Url.py | 45 ++-- bin/WebStats.py | 62 +++--- files/protocolsfile | 103 ++++++++- files/tldsfile | 263 +++++++++++++++++++++++ var/www/Flask_server.py | 9 + var/www/templates/Protocolstrending.html | 10 +- var/www/templates/Tldstrending.html | 196 +++++++++++++++++ var/www/templates/Wordstrending.html | 2 +- var/www/templates/index.html | 2 +- var/www/templates/search.html | 2 +- 11 files changed, 634 insertions(+), 66 deletions(-) create mode 100644 files/tldsfile create mode 100644 var/www/templates/Tldstrending.html diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 330a23ce..fc8c9ff1 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -107,11 +107,11 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x' sleep 0.1 - #screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' + screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' sleep 0.1 - #screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' + screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' sleep 0.1 @@ -139,7 +139,7 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' sleep 0.1 - #screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' + screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' } #If no params, display the help diff --git a/bin/Url.py b/bin/Url.py index d33e7e9f..ba3842af 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -7,6 +7,8 @@ import dns.exception from packages import Paste from packages import lib_refine from pubsublogger import publisher +from pyfaup.faup import Faup +import re # Country and ASN lookup from cymru.ip2asn.dns import DNSClient as ip2asn @@ -16,6 +18,12 @@ import ipaddress from Helper import Process +def avoidNone(str): + if str is None: + return "" + else: + return str + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" @@ -41,6 +49,7 @@ if __name__ == "__main__": message = p.get_from_set() prec_filename = None + faup = Faup() url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" @@ -53,33 +62,27 @@ if __name__ == "__main__": PST = Paste.Paste(filename) client = ip2asn() for x in PST.get_regex(url_regex): - scheme, credential, subdomain, domain, host, tld, \ - port, resource_path, query_string, f1, f2, f3, \ - f4 = x - domains_list.append(domain) -# p.populate_set_out(x, 'Url') - temp_x = () - for i in range(0,13): - if x[i] == '': - temp_x += ('None', ) - else: - temp_x += (x[i], ) - temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \ - temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \ - temp_f4 = temp_x + matching_url = re.search(url_regex, PST.get_p_content()) + url = matching_url.group(0) - to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \ - temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\ - temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date()) - p.populate_set_out(to_send , 'Url') - publisher.debug('{} Published'.format(x)) + to_send = "{} {}".format(url, PST._get_p_date()) + p.populate_set_out(to_send, 'Url') + + faup.decode(url) + domain = faup.get_domain() + subdomain = faup.get_subdomain() + f1 = None + + domains_list.append(domain) + + publisher.debug('{} Published'.format(url)) if f1 == "onion": print domain - hostl = unicode(subdomain+domain) + hostl = unicode(avoidNone(subdomain)+avoidNone(domain)) try: - socket.setdefaulttimeout(2) + socket.setdefaulttimeout(1) ip = socket.gethostbyname(unicode(hostl)) except: # If the resolver is not giving any IPv4 address, diff --git a/bin/WebStats.py b/bin/WebStats.py index 15508e52..5da443a8 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -5,14 +5,24 @@ """ import time +import datetime import re import redis import os +from packages import lib_words from pubsublogger import publisher from packages import Paste from Helper import Process +from pyfaup.faup import Faup - +def analyse(field_name): + field = url_parsed[field_name] + if field is not None: + prev_score = r_serv1.hget(field, date) + if prev_score is not None: + r_serv1.hset(field, date, int(prev_score) + 1) + else: + r_serv1.hset(field, date, 1) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) @@ -37,16 +47,22 @@ if __name__ == '__main__': db=p.config.get("Redis_Level_DB", "db")) # FILE CURVE SECTION # - csv_path = os.path.join(os.environ['AIL_HOME'], + csv_path_proto = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "protocolstrending_csv")) protocolsfile_path = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "protocolsfile")) + + csv_path_tld = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "tldstrending_csv")) + tldsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "tldsfile")) + faup = Faup() + generate_new_graph = False # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() - generate_new_graph = False if message is None: if generate_new_graph: @@ -55,10 +71,15 @@ if __name__ == '__main__': today = datetime.date.today() year = today.year month = today.month - lib_words.create_curve_with_word_file(r_serv1, csv_path, + + lib_words.create_curve_with_word_file(r_serv1, csv_path_proto, protocolsfile_path, year, month) + lib_words.create_curve_with_word_file(r_serv1, csv_path_tld, + tldsfile_path, year, + month) + publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue @@ -66,30 +87,9 @@ if __name__ == '__main__': else: generate_new_graph = True # Do something with the message from the queue - scheme, credential, subdomain, domain, host, tld, \ - port, resource_path, query_string, f1, f2, f3, \ - f4 , date= message.split() - - prev_score = r_serv1.hget(scheme, date) - if prev_score is not None: - r_serv1.hset(scheme, date, int(prev_score) + int(score)) - else: - r_serv1.hset(scheme, date, score) - - - - - - - - - - - - - - - - - - + url, date = message.split() + faup.decode(url) + url_parsed = faup.get() + + analyse('scheme') #Scheme analysis + analyse('tld') #Tld analysis diff --git a/files/protocolsfile b/files/protocolsfile index f36a40f5..ffece4b6 100644 --- a/files/protocolsfile +++ b/files/protocolsfile @@ -1,3 +1,100 @@ -FTP -HTTP -HTTPS +afs +file +ftp +z39.50 +z39.50r +z39.50s +vemmi +urn +nfs +dict +acap +rtspu +rtsp +rtsps +tip +pop +cid +mid +data +thismessage +service +shttp +fax +modem +tv +sip +sips +go +icap +h323 +ipp +xmlrpc.beep +xmlrpc.beeps +tftp +mupdate +pres +im +mtqp +tel +iris +iris.beep +crid +snmp +tag +wais +prospero +soap.beep +soap.beeps +telnet +gopher +cap +info +dns +ldap +dav +opaquelocktoken +msrp +msrps +dtn +imap +xmpp +iax +news +nntp +snews +sms +rsync +sieve +geo +mailto +jms +mailserver +ipn +tn3270 +ws +wss +xcon +xcon-userid +about +aaa +aaas +session +ni +nih +reload +ham +stun +stuns +turn +turns +http +https +coap +coaps +rtmfp +ipps +pkcs11 +acct +example +vnc diff --git a/files/tldsfile b/files/tldsfile new file mode 100644 index 00000000..73bff3af --- /dev/null +++ b/files/tldsfile @@ -0,0 +1,263 @@ +com +org +net +int +edu +gov +mil +arpa +ac +ad +ae +af +ag +ai +al +am +an +ao +aq +ar +as +at +au +aw +ax +az +ba +bb +bd +be +bf +bg +bh +bi +bj +bl +bm +bn +bo +bq +br +bs +bt +bv +bw +by +bz +ca +cc +cd +cf +cg +ch +ci +ck +cl +cm +cn +co +cr +cu +cv +cw +cx +cy +cz +de +dj +dk +dm +do +dz +ec +ee +eg +eh +er +es +et +eu +fi +fj +fk +fm +fo +fr +ga +gb +gd +ge +gf +gg +gh +gi +gl +gm +gn +gp +gq +gr +gs +gt +gu +gw +gy +hk +hm +hn +hr +ht +hu +id +ie +il +im +in +io +iq +ir +is +it +je +jm +jo +jp +ke +kg +kh +ki +km +kn +kp +kr +kw +ky +kz +la +lb +lc +li +lk +lr +ls +lt +lu +lv +ly +ma +mc +md +me +mf +mg +mh +mk +ml +mm +mn +mo +mp +mq +mr +ms +mt +mu +mv +mw +mx +my +mz +na +nc +ne +nf +ng +ni +nl +no +np +nr +nu +nz +om +pa +pe +pf +pg +ph +pk +pl +pm +pn +pr +ps +pt +pw +py +qa +re +ro +rs +ru +rw +sa +sb +sc +sd +se +sg +sh +si +sj +sk +sl +sm +sn +so +sr +ss +st +su +sv +sx +sy +sz +tc +td +tf +tg +th +tj +tk +tl +tm +tn +to +tp +tr +tt +tv +tw +tz +ua +ug +uk +um +us +uy +uz +va +vc +ve +vg +vi +vn +vu +wf +ws +ye +yt +za +zm +zw diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 7a1aa472..9c152cab 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -96,5 +96,14 @@ def wordstrending(): return render_template("Wordstrending.html") +@app.route("/protocolstrending/") +def protocolstrending(): + return render_template("Protocolstrending.html") + +@app.route("/tldstrending/") +def tldstrending(): + return render_template("Tldstrending.html") + + if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/var/www/templates/Protocolstrending.html b/var/www/templates/Protocolstrending.html index 795ca0ab..6183f2ff 100644 --- a/var/www/templates/Protocolstrending.html +++ b/var/www/templates/Protocolstrending.html @@ -24,8 +24,8 @@