#!/usr/bin/env python3 # -*-coding:UTF-8 -* import os import sys import re import redis import datetime import time import subprocess import requests from pyfaup.faup import Faup sys.path.append(os.environ['AIL_BIN']) from Helper import Process from pubsublogger import publisher # ======== GLOBAL VARIABLES ======== publisher.port = 6380 publisher.channel = "Script" config_section = 'Crawler' # Setup the I/O queues p = Process(config_section) accepted_services = ['onion', 'regular'] dic_regex = {} dic_regex['onion'] = "((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" re.compile(dic_regex['onion']) dic_regex['i2p'] = "((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.i2p)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" re.compile(dic_regex['i2p']) dic_regex['regular'] = dic_regex['i2p'] faup = Faup() PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) r_serv_metadata = redis.StrictRedis( host=p.config.get("ARDB_Metadata", "host"), port=p.config.getint("ARDB_Metadata", "port"), db=p.config.getint("ARDB_Metadata", "db"), decode_responses=True) r_cache = redis.StrictRedis( host=p.config.get("Redis_Cache", "host"), port=p.config.getint("Redis_Cache", "port"), db=p.config.getint("Redis_Cache", "db"), decode_responses=True) r_onion = redis.StrictRedis( host=p.config.get("ARDB_Onion", "host"), port=p.config.getint("ARDB_Onion", "port"), db=p.config.getint("ARDB_Onion", "db"), decode_responses=True) # ======== FUNCTIONS ======== def decode_val(value): if value is not None: value = value.decode() return value def load_type_blacklist(type_service): # load domains blacklist try: with open(os.path.join(os.environ['AIL_BIN'],'/torcrawler/blacklist_{}.txt'.format(type_service)), 'r') as f: # # TODO: # FIXME: remove this r_onion.delete('blacklist_{}'.format(type_service)) lines = f.read().splitlines() for line in lines: r_onion.sadd('blacklist_{}'.format(type_service), line) except Exception: pass def on_error_send_message_back_in_queue(type_hidden_service, domain, message): # send this msg back in the queue if not r_onion.sismember('{}_domain_crawler_queue'.format(type_hidden_service), domain): r_onion.sadd('{}_domain_crawler_queue'.format(type_hidden_service), domain) r_onion.sadd('{}_crawler_priority_queue'.format(type_hidden_service), message) def crawl_onion(url, domain, date, date_month, message): r_cache.hset('metadata_crawler:{}'.format(splash_port), 'crawling_domain', domain) r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S")) #if not r_onion.sismember('full_onion_up', domain) and not r_onion.sismember('onion_down:'+date , domain): super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father') if super_father is None: super_father=paste retry = True nb_retry = 0 while retry: try: r = requests.get(splash_url , timeout=30.0) retry = False except Exception: # TODO: relaunch docker or send error message nb_retry += 1 if nb_retry == 6: on_error_send_message_back_in_queue(type_hidden_service, domain, message) publisher.error('{} SPASH DOWN'.format(splash_url)) print('--------------------------------------') print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' {} DOWN'.format(splash_url)) r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'SPLASH DOWN') nb_retry == 0 print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m') print(' Retry({}) in 10 seconds'.format(nb_retry)) time.sleep(10) if r.status_code == 200: r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling') process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father], stdout=subprocess.PIPE) while process.poll() is None: time.sleep(1) if process.returncode == 0: output = process.stdout.read().decode() print(output) # error: splash:Connection to proxy refused if 'Connection to proxy refused' in output: on_error_send_message_back_in_queue(type_hidden_service, domain, message) publisher.error('{} SPASH, PROXY DOWN OR BAD CONFIGURATION'.format(splash_url)) print('------------------------------------------------------------------------') print(' \033[91m SPLASH: Connection to proxy refused') print('') print(' PROXY DOWN OR BAD CONFIGURATION\033[0m'.format(splash_url)) print('------------------------------------------------------------------------') r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Error') exit(-2) else: print(process.stdout.read()) exit(-1) else: on_error_send_message_back_in_queue(type_hidden_service, domain, message) print('--------------------------------------') print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' {} DOWN'.format(splash_url)) r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling') exit(1) # ======== MAIN ======== if __name__ == '__main__': if len(sys.argv) != 3: #print('usage:', 'Crawler.py', 'type_hidden_service (onion or i2p or regular)', 'splash_port') print('usage:', 'Crawler.py', 'mode (manual or automatic)', 'splash_port') exit(1) mode = sys.argv[1] splash_port = sys.argv[2] if mode == 'automatic': type_hidden_service = 'onion' # verify crawler type (type_hidden_service) if type_hidden_service not in accepted_services: print('incorrect crawler type: {}'.format(type_hidden_service)) exit(0) else: publisher.info("Script Crawler started") # load domains blacklist load_type_blacklist(type_hidden_service) splash_url = '{}:{}'.format( p.config.get("Crawler", "splash_url_onion"), splash_port) print('splash url: {}'.format(splash_url)) crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit") # Crawler status r_cache.sadd('all_crawler:{}'.format(type_hidden_service), splash_port) r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting') r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S")) while True: if mode == 'automatic': # Priority Queue - Recovering the streamed message informations. message = r_onion.spop('{}_crawler_priority_queue'.format(type_hidden_service)) if message is None: # Recovering the streamed message informations. message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) else: pass if message is not None: splitted = message.split(';') if len(splitted) == 2: url, paste = splitted paste = paste.replace(PASTES_FOLDER+'/', '') # extract data from url faup.decode(url) url_unpack = faup.get() url = decode_val(url_unpack['url']) port = decode_val(url_unpack['port']) scheme = decode_val(url_unpack['scheme']) domain = decode_val(url_unpack['domain']) host = decode_val(url_unpack['domain']) # Add Scheme to url if scheme is None: url= 'http://{}'.format(url) domain_url = 'http://{}'.format(domain) # remove url to crawl from queue r_onion.srem('{}_domain_crawler_queue'.format(type_hidden_service), domain) print() print() print('\033[92m------------------START CRAWLER------------------\033[0m') print('crawler type: {}'.format(type_hidden_service)) print('\033[92m-------------------------------------------------\033[0m') print('url: {}'.format(url)) print('domain: {}'.format(domain)) print('domain_url: {}'.format(domain_url)) if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain): date = datetime.datetime.now().strftime("%Y%m%d") date_month = datetime.datetime.now().strftime("%Y%m") if not r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and not r_onion.sismember('{}_down:{}'.format(type_hidden_service, date), domain): # first seen if not r_onion.hexists('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen'): r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date) # last_father r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'paste_parent', paste) # last check r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date) # Launch Scrapy-Splash Crawler crawl_onion(url, domain, date, date_month, message) # Crawl Domain if url != domain_url: #Crawl Domain with port number if port is not None: print('{}:{}'.format(domain_url, port)) crawl_onion('{}:{}'.format(domain_url, port), domain, date, date_month, message) #Crawl without port number print(domain_url) crawl_onion(domain_url, domain, date, date_month, message) # update last check r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date) # save down onion if not r_onion.sismember('{}_up:{}'.format(type_hidden_service, date), domain): r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain) else: #r_onion.hincrby('{}_link_up'.format(type_hidden_service), url, 1) if r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and r_serv_metadata.exists('paste_children:'+paste): msg = 'infoleak:automatic-detection="{}";{}'.format(type_hidden_service, paste) p.populate_set_out(msg, 'Tags') # add onion screenshot history # add crawled days if r_onion.lindex('{}_history:{}'.format(type_hidden_service, domain), 0) != date: r_onion.lpush('{}_history:{}'.format(type_hidden_service, domain), date) # add crawled history by date r_onion.lpush('{}_history:{}:{}'.format(type_hidden_service, domain, date), paste) if mode == 'automatic': # check external onions links (full_crawl) external_domains = set() for link in r_onion.smembers('domain_{}_external_links:{}'.format(type_hidden_service, domain)): external_domain = re.findall(dic_regex[type_hidden_service], link) external_domain.extend(re.findall(url_i2p, link)) if len(external_domain) > 0: external_domain = external_domain[0][4] else: continue if '.onion' in external_domain and external_domain != domain: external_domains.add(external_domain) elif '.i2p' in external_domain and external_domain != domain: external_domains.add(external_domain) if len(external_domains) >= 10: r_onion.sadd('{}_potential_source'.format(type_hidden_service), domain) r_onion.delete('domain_{}_external_links:{}'.format(type_hidden_service, domain)) print(r_onion.smembers('domain_{}_external_links:{}'.format(type_hidden_service, domain))) # update list, last crawled sites r_onion.lpush('last_{}'.format(type_hidden_service), domain) r_onion.ltrim('last_{}'.format(type_hidden_service), 0, 15) #update crawler status r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting') r_cache.hdel('metadata_crawler:{}'.format(splash_port), 'crawling_domain') else: print(' Blacklisted Site') print() print() else: continue else: time.sleep(1)