#!/usr/bin/env python3 # -*-coding:UTF-8 -* import os import sys import re import redis import datetime import time import subprocess import requests sys.path.append(os.environ['AIL_BIN']) from Helper import Process from pubsublogger import publisher def signal_handler(sig, frame): sys.exit(0) def crawl_onion(url, domain, date, date_month): #if not r_onion.sismember('full_onion_up', domain) and not r_onion.sismember('onion_down:'+date , domain): super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father') if super_father is None: super_father=paste try: r = requests.get(splash_url , timeout=30.0) except Exception: ## FIXME: # TODO: relaunch docker or send error message print('--------------------------------------') print(' DOCKER SPLASH DOWN') exit(0) if r.status_code == 200: process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, http_proxy, type_hidden_service, url, domain, paste, super_father], stdout=subprocess.PIPE) while process.poll() is None: time.sleep(1) if process.returncode == 0: if r_serv_metadata.exists('paste_children:'+paste): msg = 'infoleak:automatic-detection="{}";{}'.format(type_hidden_service, paste) p.populate_set_out(msg, 'Tags') print(process.stdout.read()) else: r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain) r_onion.sadd('{}_down_link:{}'.format(type_hidden_service, date), url) print(process.stdout.read()) else: ## FIXME: # TODO: relaunch docker exit(0) if __name__ == '__main__': publisher.port = 6380 publisher.channel = "Script" publisher.info("Script Crawler started") config_section = 'Crawler' # Setup the I/O queues p = Process(config_section) url_onion = "((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" re.compile(url_onion) url_i2p = "((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.i2p)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" re.compile(url_i2p) type_hidden_service = 'onion' if type_hidden_service == 'onion': regex_hidden_service = url_onion splash_url = p.config.get("Crawler", "splash_url_onion") http_proxy = p.config.get("Crawler", "http_proxy_onion") elif type_hidden_service == 'i2p': regex_hidden_service = url_i2p splash_url = p.config.get("Crawler", "splash_url_i2p") http_proxy = p.config.get("Crawler", "http_proxy_i2p") elif type_hidden_service == 'regular': regex_hidden_service = url_i2p splash_url = p.config.get("Crawler", "splash_url_onion") http_proxy = p.config.get("Crawler", "http_proxy_onion") else: print('incorrect crawler type: {}'.format(type_hidden_service)) exit(0) crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit") #signal.signal(signal.SIGINT, signal_handler) r_serv_metadata = redis.StrictRedis( host=p.config.get("ARDB_Metadata", "host"), port=p.config.getint("ARDB_Metadata", "port"), db=p.config.getint("ARDB_Metadata", "db"), decode_responses=True) r_cache = redis.StrictRedis( host=p.config.get("Redis_Cache", "host"), port=p.config.getint("Redis_Cache", "port"), db=p.config.getint("Redis_Cache", "db"), decode_responses=True) r_onion = redis.StrictRedis( host=p.config.get("ARDB_Onion", "host"), port=p.config.getint("ARDB_Onion", "port"), db=p.config.getint("ARDB_Onion", "db"), decode_responses=True) while True: # Recovering the streamed message informations. message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) # # FIXME: remove if message is None: print('get ardb message') message = r_onion.spop('mess_onion') if message is not None: splitted = message.split(';') if len(splitted) == 2: url, paste = splitted if not '.onion' in url: print('not onion') continue url_list = re.findall(regex_hidden_service, url)[0] if url_list[1] == '': url= 'http://{}'.format(url) link, s, credential, subdomain, domain, host, port, \ resource_path, query_string, f1, f2, f3, f4 = url_list domain = url_list[4] r_onion.srem('onion_domain_crawler_queue', domain) domain_url = 'http://{}'.format(domain) print('------------------START CRAWLER------------------') print(type_hidden_service) print('-------------------------------------------------') print('url: {}'.format(url)) print('domain: {}'.format(domain)) print('domain_url: {}'.format(domain_url)) if not r_onion.sismember('banned_{}'.format(type_hidden_service), domain): date = datetime.datetime.now().strftime("%Y%m%d") date_month = datetime.datetime.now().strftime("%Y%m") if not r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and not r_onion.sismember('{}_down:{}'.format(type_hidden_service, date), domain): crawl_onion(url, domain, date, date_month) if url != domain_url: crawl_onion(domain_url, domain, date, date_month) # save down onion if not r_onion.sismember('{}_up:{}'.format(type_hidden_service, date), domain): r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain) r_onion.sadd('{}_down_link:{}'.format(type_hidden_service, date), url) r_onion.hincrby('{}_link_down'.format(type_hidden_service), url, 1) if not r_onion.exists('{}_metadata:{}'.format(type_hidden_service, domain)): r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date) r_onion.hset('{}_metadata:{}'.format(type_hidden_service,domain), 'last_seen', date) else: r_onion.hincrby('{}_link_up'.format(type_hidden_service), url, 1) # last check r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date) # check external onions links (full_scrawl) external_domains = set() for link in r_onion.smembers('domain_{}_external_links:{}'.format(type_hidden_service, domain)): external_domain = re.findall(url_onion, link) external_domain.extend(re.findall(url_i2p, link)) if len(external_domain) > 0: external_domain = external_domain[0][4] else: continue if '.onion' in external_domain and external_domain != domain: external_domains.add(external_domain) elif '.i2p' in external_domain and external_domain != domain: external_domains.add(external_domain) if len(external_domains) >= 10: r_onion.sadd('{}_potential_source'.format(type_hidden_service), domain) r_onion.delete('domain_{}_external_links:{}'.format(type_hidden_service, domain)) print(r_onion.smembers('domain_{}_external_links:{}'.format(type_hidden_service, domain))) r_onion.lpush('last_{}'.format(type_hidden_service), domain) r_onion.ltrim('last_{}'.format(type_hidden_service), 0, 15) else: continue else: time.sleep(1)