mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
117 lines
4 KiB
Python
Executable file
117 lines
4 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import redis
|
|
import datetime
|
|
import time
|
|
import subprocess
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
from Helper import Process
|
|
from pubsublogger import publisher
|
|
|
|
|
|
def signal_handler(sig, frame):
|
|
sys.exit(0)
|
|
|
|
def crawl_onion(url, domain):
|
|
date = datetime.datetime.now().strftime("%Y%m%d")
|
|
|
|
if not r_onion.sismember('onion_up:'+date , domain):
|
|
super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father')
|
|
if super_father is None:
|
|
super_father=paste
|
|
|
|
process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', url, domain, paste, super_father],
|
|
stdout=subprocess.PIPE)
|
|
while process.poll() is None:
|
|
time.sleep(1)
|
|
|
|
if process.returncode == 0:
|
|
if r_serv_metadata.exists('paste_children:'+paste):
|
|
msg = 'infoleak:automatic-detection="onion";{}'.format(paste)
|
|
p.populate_set_out(msg, 'Tags')
|
|
print(process.stdout.read())
|
|
|
|
r_onion.sadd('onion_up:'+date , domain)
|
|
r_onion.sadd('onion_up_link:'+date , url)
|
|
else:
|
|
r_onion.sadd('onion_down:'+date , domain)
|
|
r_onion.sadd('onion_down_link:'+date , url)
|
|
print(process.stdout.read())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
publisher.port = 6380
|
|
publisher.channel = "Script"
|
|
|
|
publisher.info("Script Crawler started")
|
|
|
|
config_section = 'Crawler'
|
|
|
|
# Setup the I/O queues
|
|
p = Process(config_section)
|
|
|
|
splash_url = p.config.get("Crawler", "splash_url")
|
|
http_proxy = p.config.get("Crawler", "http_proxy")
|
|
crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit")
|
|
|
|
#signal.signal(signal.SIGINT, signal_handler)
|
|
|
|
r_serv_metadata = redis.StrictRedis(
|
|
host=p.config.get("ARDB_Metadata", "host"),
|
|
port=p.config.getint("ARDB_Metadata", "port"),
|
|
db=p.config.getint("ARDB_Metadata", "db"),
|
|
decode_responses=True)
|
|
|
|
r_cache = redis.StrictRedis(
|
|
host=p.config.get("Redis_Cache", "host"),
|
|
port=p.config.getint("Redis_Cache", "port"),
|
|
db=p.config.getint("Redis_Cache", "db"),
|
|
decode_responses=True)
|
|
|
|
r_onion = redis.StrictRedis(
|
|
host=p.config.get("ARDB_Onion", "host"),
|
|
port=p.config.getint("ARDB_Onion", "port"),
|
|
db=p.config.getint("ARDB_Onion", "db"),
|
|
decode_responses=True)
|
|
|
|
url_regex = "((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)"
|
|
re.compile(url_regex)
|
|
|
|
while True:
|
|
|
|
message = p.get_from_set()
|
|
# Recovering the streamed message informations.
|
|
if message is not None:
|
|
splitted = message.split(';')
|
|
if len(splitted) == 2:
|
|
url, paste = splitted
|
|
|
|
url_list = re.findall(url_regex, url)[0]
|
|
if url_list[1] == '':
|
|
url= 'http://{}'.format(url)
|
|
|
|
link, s, credential, subdomain, domain, host, port, \
|
|
resource_path, query_string, f1, f2, f3, f4 = url_list
|
|
domain = url_list[4]
|
|
|
|
domain_url = 'http://{}'.format(domain)
|
|
|
|
print('------------------START ONIOM CRAWLER------------------')
|
|
print('url: {}'.format(url))
|
|
print('domain: {}'.format(domain))
|
|
print('domain_url: {}'.format(domain_url))
|
|
|
|
crawl_onion(url, domain)
|
|
if url != domain_url:
|
|
crawl_onion(domain_url, domain)
|
|
|
|
else:
|
|
continue
|
|
else:
|
|
time.sleep(1)
|