+
+
+
@@ -129,6 +136,8 @@
+
+ | Latest Release | - |
+ |
| Contributors | @@ -168,6 +168,22 @@ Privacy and GDPR [AIL information leaks analysis and the GDPR in the context of collection, analysis and sharing information leaks](https://www.circl.lu/assets/files/information-leaks-analysis-and-gdpr.pdf) document provides an overview how to use AIL in a lawfulness context especially in the scope of General Data Protection Regulation. +Research using AIL +------------------ + +If you write academic paper, relying or using AIL, it can be cited with the following BibTeX: + +~~~~ +@inproceedings{mokaddem2018ail, + title={AIL-The design and implementation of an Analysis Information Leak framework}, + author={Mokaddem, Sami and Wagener, G{\'e}rard and Dulaunoy, Alexandre}, + booktitle={2018 IEEE International Conference on Big Data (Big Data)}, + pages={5049--5057}, + year={2018}, + organization={IEEE} +} +~~~~ + Screenshots =========== @@ -237,11 +253,11 @@ License ``` Copyright (C) 2014 Jules Debra - Copyright (C) 2014-2018 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) - Copyright (c) 2014-2018 Raphaël Vinot - Copyright (c) 2014-2018 Alexandre Dulaunoy - Copyright (c) 2016-2018 Sami Mokaddem - Copyright (c) 2018 Thirion Aurélien + Copyright (C) 2014-2019 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) + Copyright (c) 2014-2019 Raphaël Vinot + Copyright (c) 2014-2019 Alexandre Dulaunoy + Copyright (c) 2016-2019 Sami Mokaddem + Copyright (c) 2018-2019 Thirion Aurélien This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/bin/CVE_check.py b/bin/CVE_check.py new file mode 100755 index 00000000..63f611de --- /dev/null +++ b/bin/CVE_check.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +from packages import Paste +from Helper import Process + +import os +import re +import time +import redis +import configparser + +from collections import defaultdict + +def get_dict_cve(list_paste_cve, only_one_same_cve_by_paste=False): + dict_keyword = {} + + for paste_cve in list_paste_cve: + paste_content = Paste.Paste(paste_cve).get_p_content() + + cve_list = reg_cve.findall(paste_content) + if only_one_same_cve_by_paste: + cve_list = set(cve_list) + + for cve in reg_cve.findall(paste_content): + try: + dict_keyword[cve] += 1 + except KeyError: + dict_keyword[cve] = 1 + + print('------------------------------------------------') + if dict_keyword: + res = [(k, dict_keyword[k]) for k in sorted(dict_keyword, key=dict_keyword.get, reverse=True)] + for item in res: + pass + print(item) + + + +if __name__ == '__main__': + + # CONFIG # + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + cfg = configparser.ConfigParser() + cfg.read(configfile) + + serv_metadata = redis.StrictRedis( + host=cfg.get("ARDB_Metadata", "host"), + port=cfg.getint("ARDB_Metadata", "port"), + db=cfg.getint("ARDB_Metadata", "db"), + decode_responses=True) + + serv_tags = redis.StrictRedis( + host=cfg.get("ARDB_Tags", "host"), + port=cfg.get("ARDB_Tags", "port"), + db=cfg.get("ARDB_Tags", "db"), + decode_responses=True) + + reg_cve = re.compile(r'CVE-[1-2]\d{1,4}-\d{1,7}') + + #all_past_cve = serv_tags.smembers('infoleak:automatic-detection="cve"') + #all_past_cve_regular = serv_tags.sdiff('infoleak:automatic-detection="cve"', 'infoleak:submission="crawler"') + #all_past_cve_crawler = serv_tags.sinter('infoleak:automatic-detection="cve"', 'infoleak:submission="crawler"') + + #print('{} + {} = {}'.format(len(all_past_cve_regular), len(all_past_cve_crawler), len(all_past_cve))) + + print('ALL_CVE') + get_dict_cve(serv_tags.smembers('infoleak:automatic-detection="cve"'), True) + print() + print() + print() + print('REGULAR_CVE') + get_dict_cve(serv_tags.sdiff('infoleak:automatic-detection="cve"', 'infoleak:submission="crawler"'), True) + print() + print() + print() + print('CRAWLER_CVE') + get_dict_cve(serv_tags.sinter('infoleak:automatic-detection="cve"', 'infoleak:submission="crawler"'), True) diff --git a/bin/Crawler.py b/bin/Crawler.py index 0f69cfe6..e6b61a99 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -10,6 +10,8 @@ import time import subprocess import requests +from pyfaup.faup import Faup + sys.path.append(os.environ['AIL_BIN']) from Helper import Process from pubsublogger import publisher @@ -18,10 +20,13 @@ def on_error_send_message_back_in_queue(type_hidden_service, domain, message): # send this msg back in the queue if not r_onion.sismember('{}_domain_crawler_queue'.format(type_hidden_service), domain): r_onion.sadd('{}_domain_crawler_queue'.format(type_hidden_service), domain) - r_onion.sadd('{}_crawler_queue'.format(type_hidden_service), message) + r_onion.sadd('{}_crawler_priority_queue'.format(type_hidden_service), message) def crawl_onion(url, domain, date, date_month, message): + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'crawling_domain', domain) + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S")) + #if not r_onion.sismember('full_onion_up', domain) and not r_onion.sismember('onion_down:'+date , domain): super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father') if super_father is None: @@ -37,19 +42,21 @@ def crawl_onion(url, domain, date, date_month, message): # TODO: relaunch docker or send error message nb_retry += 1 - if nb_retry == 30: + if nb_retry == 6: on_error_send_message_back_in_queue(type_hidden_service, domain, message) publisher.error('{} SPASH DOWN'.format(splash_url)) print('--------------------------------------') print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' {} DOWN'.format(splash_url)) - exit(1) + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'SPLASH DOWN') + nb_retry == 0 print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m') print(' Retry({}) in 10 seconds'.format(nb_retry)) time.sleep(10) if r.status_code == 200: + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling') process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father], stdout=subprocess.PIPE) while process.poll() is None: @@ -67,6 +74,7 @@ def crawl_onion(url, domain, date, date_month, message): print('') print(' PROXY DOWN OR BAD CONFIGURATION\033[0m'.format(splash_url)) print('------------------------------------------------------------------------') + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Error') exit(-2) else: print(process.stdout.read()) @@ -76,6 +84,7 @@ def crawl_onion(url, domain, date, date_month, message): print('--------------------------------------') print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' {} DOWN'.format(splash_url)) + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling') exit(1) @@ -119,6 +128,7 @@ if __name__ == '__main__': print('splash url: {}'.format(splash_url)) crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit") + faup = Faup() PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) @@ -140,6 +150,10 @@ if __name__ == '__main__': db=p.config.getint("ARDB_Onion", "db"), decode_responses=True) + r_cache.sadd('all_crawler:{}'.format(type_hidden_service), splash_port) + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting') + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S")) + # load domains blacklist try: with open(os.environ['AIL_BIN']+'/torcrawler/blacklist_onion.txt', 'r') as f: @@ -152,8 +166,12 @@ if __name__ == '__main__': while True: - # Recovering the streamed message informations. - message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) + # Priority Queue - Recovering the streamed message informations. + message = r_onion.spop('{}_crawler_priority_queue'.format(type_hidden_service)) + + if message is None: + # Recovering the streamed message informations. + message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) if message is not None: @@ -173,6 +191,8 @@ if __name__ == '__main__': domain_url = 'http://{}'.format(domain) + print() + print() print('\033[92m------------------START CRAWLER------------------\033[0m') print('crawler type: {}'.format(type_hidden_service)) print('\033[92m-------------------------------------------------\033[0m') @@ -180,12 +200,24 @@ if __name__ == '__main__': print('domain: {}'.format(domain)) print('domain_url: {}'.format(domain_url)) - if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain): + faup.decode(domain) + onion_domain=faup.get()['domain'].decode() + + if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain) and not r_onion.sismember('blacklist_{}'.format(type_hidden_service), onion_domain): date = datetime.datetime.now().strftime("%Y%m%d") date_month = datetime.datetime.now().strftime("%Y%m") if not r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and not r_onion.sismember('{}_down:{}'.format(type_hidden_service, date), domain): + # first seen + if not r_onion.hexists('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen'): + r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date) + + # last_father + r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'paste_parent', paste) + + # last check + r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date) crawl_onion(url, domain, date, date_month, message) if url != domain_url: @@ -198,21 +230,12 @@ if __name__ == '__main__': r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain) #r_onion.sadd('{}_down_link:{}'.format(type_hidden_service, date), url) #r_onion.hincrby('{}_link_down'.format(type_hidden_service), url, 1) - if not r_onion.exists('{}_metadata:{}'.format(type_hidden_service, domain)): - r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date) - r_onion.hset('{}_metadata:{}'.format(type_hidden_service,domain), 'last_seen', date) else: #r_onion.hincrby('{}_link_up'.format(type_hidden_service), url, 1) if r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and r_serv_metadata.exists('paste_children:'+paste): msg = 'infoleak:automatic-detection="{}";{}'.format(type_hidden_service, paste) p.populate_set_out(msg, 'Tags') - # last check - r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date) - - # last_father - r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'paste_parent', paste) - # add onion screenshot history # add crawled days if r_onion.lindex('{}_history:{}'.format(type_hidden_service, domain), 0) != date: @@ -243,6 +266,14 @@ if __name__ == '__main__': r_onion.lpush('last_{}'.format(type_hidden_service), domain) r_onion.ltrim('last_{}'.format(type_hidden_service), 0, 15) + #update crawler status + r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting') + r_cache.hdel('metadata_crawler:{}'.format(splash_port), 'crawling_domain') + else: + print(' Blacklisted Onion') + print() + print() + else: continue else: diff --git a/bin/Global.py b/bin/Global.py index c1e16496..6f26ad0e 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -23,23 +23,17 @@ Requirements import base64 import os import time +import uuid from pubsublogger import publisher from Helper import Process import magic -import io -#import gzip -''' -def gunzip_bytes_obj(bytes_obj): - in_ = io.BytesIO() - in_.write(bytes_obj) - in_.seek(0) - with gzip.GzipFile(fileobj=in_, mode='rb') as fo: - gunzipped_bytes_obj = fo.read() +def rreplace(s, old, new, occurrence): + li = s.rsplit(old, occurrence) + return new.join(li) - return gunzipped_bytes_obj.decode()''' if __name__ == '__main__': publisher.port = 6380 @@ -79,6 +73,12 @@ if __name__ == '__main__': processed_paste = 0 time.sleep(1) continue + + file_name_paste = paste.split('/')[-1] + if len(file_name_paste)>255: + new_file_name_paste = '{}{}.gz'.format(file_name_paste[:215], str(uuid.uuid4())) + paste = rreplace(paste, file_name_paste, new_file_name_paste, 1) + # Creating the full filepath filename = os.path.join(PASTES_FOLDER, paste) diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py index 30a24f15..cef6301c 100755 --- a/bin/ModulesInformationV2.py +++ b/bin/ModulesInformationV2.py @@ -31,7 +31,7 @@ lastTimeKillCommand = {} current_selected_value = 0 current_selected_queue = "" current_selected_action = "" -current_selected_action = 0 +current_selected_amount = 0 # Map PID to Queue name (For restart and killing) PID_NAME_DICO = {} @@ -480,7 +480,10 @@ class Show_paste(Frame): self.label_list[i]._text = "" except Exception as e: - self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value] + if current_selected_value in COMPLETE_PASTE_PATH_PER_PID: + self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value] + else: + self.label_list[0]._text = "Error Generic exception caught" self.label_list[1]._text = str(e) for i in range(2,self.num_label): self.label_list[i]._text = "" diff --git a/bin/Onion.py b/bin/Onion.py index e38f363a..ddae9afc 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -29,10 +29,18 @@ import os import base64 import subprocess import redis +import signal import re from Helper import Process +class TimeoutException(Exception): + pass + +def timeout_handler(signum, frame): + raise TimeoutException + +signal.signal(signal.SIGALRM, timeout_handler) def fetch(p, r_cache, urls, domains, path): failed = [] @@ -113,6 +121,8 @@ if __name__ == "__main__": message = p.get_from_set() prec_filename = None + max_execution_time = p.config.getint("Onion", "max_execution_time") + # send to crawler: activate_crawler = p.config.get("Crawler", "activate_crawler") if activate_crawler == 'True': @@ -130,6 +140,7 @@ if __name__ == "__main__": while True: + message = p.get_from_set() if message is not None: print(message) filename, score = message.split() @@ -140,16 +151,26 @@ if __name__ == "__main__": urls = [] PST = Paste.Paste(filename) - for x in PST.get_regex(url_regex): - print(x) - # Extracting url with regex - url, s, credential, subdomain, domain, host, port, \ - resource_path, query_string, f1, f2, f3, f4 = x + # max execution time on regex + signal.alarm(max_execution_time) + try: + for x in PST.get_regex(url_regex): + print(x) + # Extracting url with regex + url, s, credential, subdomain, domain, host, port, \ + resource_path, query_string, f1, f2, f3, f4 = x - if '.onion' in url: - print(url) - domains_list.append(domain) - urls.append(url) + if '.onion' in url: + print(url) + domains_list.append(domain) + urls.append(url) + except TimeoutException: + encoded_list = [] + p.incr_module_timeout_statistic() + print ("{0} processing timeout".format(PST.p_path)) + continue + + signal.alarm(0) ''' for x in PST.get_regex(i2p_regex): @@ -177,8 +198,12 @@ if __name__ == "__main__": print(len(domains_list)) if len(domains_list) > 0: - publisher.warning('{}Detected {} .onion(s);{}'.format( - to_print, len(domains_list),PST.p_rel_path)) + if not activate_crawler: + publisher.warning('{}Detected {} .onion(s);{}'.format( + to_print, len(domains_list),PST.p_rel_path)) + else: + publisher.info('{}Detected {} .onion(s);{}'.format( + to_print, len(domains_list),PST.p_rel_path)) now = datetime.datetime.now() path = os.path.join('onions', str(now.year).zfill(4), str(now.month).zfill(2), @@ -199,12 +224,20 @@ if __name__ == "__main__": else: continue + # too many subdomain + if len(domain.split('.')) > 5: + continue + if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain): if not r_onion.sismember('onion_domain_crawler_queue', domain): print('send to onion crawler') r_onion.sadd('onion_domain_crawler_queue', domain) msg = '{};{}'.format(url,PST.p_rel_path) - r_onion.sadd('onion_crawler_queue', msg) + if not r_onion.hexists('onion_metadata:{}'.format(domain), 'first_seen'): + r_onion.sadd('onion_crawler_priority_queue', msg) + print('send to priority queue') + else: + r_onion.sadd('onion_crawler_queue', msg) #p.populate_set_out(msg, 'Crawler') else: @@ -222,4 +255,3 @@ if __name__ == "__main__": publisher.debug("Script url is Idling 10s") #print('Sleeping') time.sleep(10) - message = p.get_from_set() diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index 280849ba..5c9f743c 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -67,7 +67,7 @@ while True: print(paste) with open(pystemonpath+paste, 'rb') as f: #.read() messagedata = f.read() - path_to_send = pastes_directory+paste + path_to_send = os.path.join(pastes_directory,paste) s = b' '.join( [ topic.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] ) socket.send(s) diff --git a/bin/packages/config.cfg.docker-compose-sample b/bin/packages/config.cfg.docker-compose-sample new file mode 100644 index 00000000..2f563493 --- /dev/null +++ b/bin/packages/config.cfg.docker-compose-sample @@ -0,0 +1,253 @@ +[Directories] +bloomfilters = Blooms +dicofilters = Dicos +pastes = PASTES +hash = HASHS +crawled = crawled +crawled_screenshot = CRAWLED_SCREENSHOT + +wordtrending_csv = var/www/static/csv/wordstrendingdata +wordsfile = files/wordfile + +protocolstrending_csv = var/www/static/csv/protocolstrendingdata +protocolsfile = files/protocolsfile + +tldstrending_csv = var/www/static/csv/tldstrendingdata +tldsfile = faup/src/data/mozilla.tlds + +domainstrending_csv = var/www/static/csv/domainstrendingdata + +pystemonpath = /opt/pystemon/ + +sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt + +##### Notifications ###### +[Notifications] +ail_domain = http://localhost:7000 +sender = sender@example.com +sender_host = smtp.example.com +sender_port = 1337 +sender_pw = None + +# optional for using with authenticated SMTP over SSL +# sender_pw = securepassword + +##### Flask ##### +[Flask] +#Proxying requests to the app +baseUrl = / +#Number of logs to display in the dashboard +max_dashboard_logs = 15 +#Maximum number of character to display in the toolip +max_preview_char = 250 +#Maximum number of character to display in the modal +max_preview_modal = 800 +#Default number of header to display in trending graphs +default_display = 10 +#Number of minutes displayed for the number of processed pastes. +minute_processed_paste = 10 +#Maximum line length authorized to make a diff between duplicates +DiffMaxLineLength = 10000 + +#### Modules #### +[BankAccount] +max_execution_time = 60 + +[Categ] +#Minimum number of match between the paste and the category file +matchingThreshold=1 + +[Credential] +#Minimum length that a credential must have to be considered as such +minimumLengthThreshold=3 +#Will be pushed as alert if the number of credentials is greater to that number +criticalNumberToAlert=8 +#Will be considered as false positive if less that X matches from the top password list +minTopPassList=5 + +[Curve] +max_execution_time = 90 + +[Onion] +max_execution_time = 180 + +[Base64] +path = Base64/ +max_execution_time = 60 + +[Binary] +path = Base64/ +max_execution_time = 60 + +[Hex] +path = Base64/ +max_execution_time = 60 + +[Modules_Duplicates] +#Number of month to look back +maximum_month_range = 3 +#The value where two pastes are considerate duplicate for ssdeep. +threshold_duplicate_ssdeep = 50 +#The value where two pastes are considerate duplicate for tlsh. +threshold_duplicate_tlsh = 52 +#Minimum size of the paste considered +min_paste_size = 0.3 + +[Module_ModuleInformation] +#Threshold to deduce if a module is stuck or not, in seconds. +threshold_stucked_module=600 + +[Module_Mixer] +#Define the configuration of the mixer, possible value: 1, 2 or 3 +operation_mode = 3 +#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) +ttl_duplicate = 86400 +default_unnamed_feed_name = unnamed_feeder + +[RegexForTermsFrequency] +max_execution_time = 60 + +##### Redis ##### +[Redis_Cache] +host = localhost +port = 6379 +db = 0 + +[Redis_Log] +host = localhost +port = 6380 +db = 0 + +[Redis_Log_submit] +host = localhost +port = 6380 +db = 1 + +[Redis_Queues] +host = localhost +port = 6381 +db = 0 + +[Redis_Data_Merging] +host = localhost +port = 6379 +db = 1 + +[Redis_Paste_Name] +host = localhost +port = 6379 +db = 2 + +[Redis_Mixer_Cache] +host = localhost +port = 6381 +db = 1 + +##### ARDB ##### +[ARDB_Curve] +host = localhost +port = 6382 +db = 1 + +[ARDB_Sentiment] +host = localhost +port = 6382 +db = 4 + +[ARDB_TermFreq] +host = localhost +port = 6382 +db = 2 + +[ARDB_TermCred] +host = localhost +port = 6382 +db = 5 + +[ARDB_DB] +host = localhost +port = 6382 +db = 0 + +[ARDB_Trending] +host = localhost +port = 6382 +db = 3 + +[ARDB_Hashs] +host = localhost +db = 1 + +[ARDB_Tags] +host = localhost +port = 6382 +db = 6 + +[ARDB_Metadata] +host = localhost +port = 6382 +db = 7 + +[ARDB_Statistics] +host = localhost +port = 6382 +db = 8 + +[ARDB_Onion] +host = localhost +port = 6382 +db = 9 + +[Url] +cc_critical = DE + +[DomClassifier] +cc = DE +cc_tld = r'\.de$' +dns = 8.8.8.8 + +[Mail] +dns = 8.8.8.8 + +[Web] +dns = 149.13.33.69 + +# Indexer configuration +[Indexer] +type = whoosh +path = indexdir +register = indexdir/all_index.txt +#size in Mb +index_max_size = 2000 + +[ailleakObject] +maxDuplicateToPushToMISP=10 + +############################################################################### + +# For multiple feed, add them with "," without space +# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557 +[ZMQ_Global] +#address = tcp://crf.circl.lu:5556 +address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 +channel = 102 +bind = tcp://127.0.0.1:5556 + +[ZMQ_Url] +address = tcp://127.0.0.1:5004 +channel = urls + +[ZMQ_FetchedOnion] +address = tcp://127.0.0.1:5005 +channel = FetchedOnion + +[RedisPubSub] +host = localhost +port = 6381 +db = 0 + +[Crawler] +activate_crawler = False +crawler_depth_limit = 1 +splash_url_onion = http://172.17.0.1 +splash_onion_port = 8050 diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index b5980766..ace656cc 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -68,6 +68,9 @@ minTopPassList=5 [Curve] max_execution_time = 90 +[Onion] +max_execution_time = 180 + [Base64] path = Base64/ max_execution_time = 60 diff --git a/bin/torcrawler/TorSplashCrawler.py b/bin/torcrawler/TorSplashCrawler.py index 47486dd9..6bb4d938 100644 --- a/bin/torcrawler/TorSplashCrawler.py +++ b/bin/torcrawler/TorSplashCrawler.py @@ -10,10 +10,12 @@ import datetime import base64 import redis import json +import time from scrapy.spidermiddlewares.httperror import HttpError from twisted.internet.error import DNSLookupError from twisted.internet.error import TimeoutError +from twisted.web._newclient import ResponseNeverReceived from scrapy import Spider from scrapy.linkextractors import LinkExtractor @@ -39,6 +41,8 @@ class TorSplashCrawler(): 'SPIDER_MIDDLEWARES': {'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,}, 'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter', 'HTTPERROR_ALLOW_ALL': True, + 'RETRY_TIMES': 2, + 'CLOSESPIDER_PAGECOUNT': 50, 'DEPTH_LIMIT': crawler_depth_limit }) @@ -97,7 +101,7 @@ class TorSplashCrawler(): yield SplashRequest( self.start_urls, self.parse, - #errback=self.errback_catcher, + errback=self.errback_catcher, endpoint='render.json', meta={'father': self.original_paste}, args={ 'html': 1, @@ -122,7 +126,11 @@ class TorSplashCrawler(): print('Connection to proxy refused') else: - UUID = self.domains[0]+str(uuid.uuid4()) + #avoid filename too big + if len(self.domains[0]) > 215: + UUID = self.domains[0][-215:]+str(uuid.uuid4()) + else: + UUID = self.domains[0]+str(uuid.uuid4()) filename_paste = os.path.join(self.crawled_paste_filemame, UUID) relative_filename_paste = os.path.join(self.crawler_path, UUID) filename_screenshot = os.path.join(self.crawled_screenshot, UUID +'.png') @@ -174,7 +182,7 @@ class TorSplashCrawler(): yield SplashRequest( link.url, self.parse, - #errback=self.errback_catcher, + errback=self.errback_catcher, endpoint='render.json', meta={'father': relative_filename_paste}, args={ 'html': 1, @@ -184,17 +192,39 @@ class TorSplashCrawler(): 'wait': 10} ) - ''' def errback_catcher(self, failure): # catch all errback failures, self.logger.error(repr(failure)) - print('failure') - #print(failure) - print(failure.type) - #print(failure.request.meta['item']) + if failure.check(ResponseNeverReceived): + request = failure.request + url = request.meta['splash']['args']['url'] + father = request.meta['father'] + + self.logger.error('Splash, ResponseNeverReceived for %s, retry in 10s ...', url) + time.sleep(10) + yield SplashRequest( + url, + self.parse, + errback=self.errback_catcher, + endpoint='render.json', + meta={'father': father}, + args={ 'html': 1, + 'png': 1, + 'render_all': 1, + 'har': 1, + 'wait': 10} + ) + + else: + print('failure') + #print(failure) + print(failure.type) + #print(failure.request.meta['item']) + + ''' #if isinstance(failure.value, HttpError): - if failure.check(HttpError): + elif failure.check(HttpError): # you can get the response response = failure.value.response print('HttpError') @@ -214,7 +244,7 @@ class TorSplashCrawler(): print('TimeoutError') print(TimeoutError) self.logger.error('TimeoutError on %s', request.url) - ''' + ''' def save_crawled_paste(self, filename, content): diff --git a/bin/torcrawler/blacklist_onion.txt b/bin/torcrawler/blacklist_onion.txt index 15dfa0de..3718fc96 100644 --- a/bin/torcrawler/blacklist_onion.txt +++ b/bin/torcrawler/blacklist_onion.txt @@ -3,3 +3,5 @@ facebookcorewwwi.onion graylady3jvrrxbe.onion expyuzz4wqqyqhjn.onion dccbbv6cooddgcrq.onion +pugljpwjhbiagkrn.onion +jld3zkuo4b5mbios.onion diff --git a/bin/torcrawler/launch_splash_crawler.sh b/bin/torcrawler/launch_splash_crawler.sh index e47efc36..5c7f21ee 100755 --- a/bin/torcrawler/launch_splash_crawler.sh +++ b/bin/torcrawler/launch_splash_crawler.sh @@ -5,12 +5,15 @@ usage() { echo "Usage: sudo $0 [-f
| + {{crawler['crawler_info']}} + | ++ {{crawler['crawling_domain']}} + | ++ {{crawler['status_info']}} + | +