mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
1500 lines
51 KiB
Python
Executable file
1500 lines
51 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
"""
|
|
API Helper
|
|
===================
|
|
|
|
|
|
"""
|
|
import base64
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pickle
|
|
import re
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
from enum import IntEnum, unique
|
|
from datetime import datetime, timedelta
|
|
from urllib.parse import urlparse, urljoin
|
|
#from bs4 import BeautifulSoup
|
|
|
|
from pylacus import PyLacus
|
|
|
|
from pyfaup.faup import Faup
|
|
|
|
# interact with splash_crawler API
|
|
import requests
|
|
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from packages import git_status
|
|
from lib.ConfigLoader import ConfigLoader
|
|
from lib.objects.Domains import Domain
|
|
from lib.objects.Items import Item
|
|
from core import screen
|
|
|
|
config_loader = ConfigLoader()
|
|
r_db = config_loader.get_db_conn("Kvrocks_DB")
|
|
r_crawler = config_loader.get_db_conn("Kvrocks_Crawler")
|
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|
|
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
|
|
|
ITEMS_FOLDER = config_loader.get_config_str("Directories", "pastes")
|
|
HAR_DIR = config_loader.get_files_directory('har')
|
|
activate_crawler = config_loader.get_config_str("Crawler", "activate_crawler")
|
|
config_loader = None
|
|
|
|
faup = Faup()
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# COMMON #
|
|
# #
|
|
# # # # # # # #
|
|
|
|
def gen_uuid():
|
|
return str(uuid.uuid4())
|
|
|
|
def generate_uuid():
|
|
return str(uuid.uuid4()).replace('-', '')
|
|
|
|
# # TODO: remove me ?
|
|
def get_current_date(separator=False):
|
|
if separator:
|
|
return datetime.now().strftime("%Y/%m/%d")
|
|
else:
|
|
return datetime.now().strftime("%Y%m%d")
|
|
|
|
def get_date_crawled_items_source(date):
|
|
return os.path.join('crawled', date)
|
|
|
|
def get_date_har_dir(date):
|
|
return os.path.join(HAR_DIR, date)
|
|
|
|
def is_valid_onion_domain(domain):
|
|
if not domain.endswith('.onion'):
|
|
return False
|
|
domain = domain.replace('.onion', '', 1)
|
|
if len(domain) == 16: # v2 address
|
|
r_onion = r'[a-z0-9]{16}'
|
|
if re.match(r_onion, domain):
|
|
return True
|
|
elif len(domain) == 56: # v3 address
|
|
r_onion = r'[a-z0-9]{56}'
|
|
if re.fullmatch(r_onion, domain):
|
|
return True
|
|
return False
|
|
|
|
def get_faup():
|
|
return faup
|
|
|
|
def unpack_url(url):
|
|
f = get_faup()
|
|
f.decode(url)
|
|
url_decoded = f.get()
|
|
port = url_decoded['port']
|
|
if not port:
|
|
if url_decoded['scheme'] == 'http':
|
|
port = 80
|
|
elif url_decoded['scheme'] == 'https':
|
|
port = 443
|
|
else:
|
|
port = 80
|
|
url_decoded['port'] = port
|
|
# decode URL
|
|
try:
|
|
url = url_decoded['url'].decode()
|
|
except AttributeError:
|
|
url = url_decoded['url']
|
|
# if not url_decoded['scheme']:
|
|
# url = f'http://{url}'
|
|
|
|
# Fix case
|
|
url_decoded['domain'] = url_decoded['domain'].lower()
|
|
url_decoded['url'] = url.replace(url_decoded['host'], url_decoded['host'].lower(), 1)
|
|
return url_decoded
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# FAVICON # TODO REWRITE ME
|
|
# #
|
|
# # # # # # # #
|
|
|
|
def get_favicon_from_html(html, domain, url):
|
|
favicon_urls = extract_favicon_from_html(html, url)
|
|
# add root favicon
|
|
if not favicon_urls:
|
|
favicon_urls.add(f'{urlparse(url).scheme}://{domain}/favicon.ico')
|
|
print(favicon_urls)
|
|
return favicon_urls
|
|
|
|
def extract_favicon_from_html(html, url):
|
|
favicon_urls = set()
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
set_icons = set()
|
|
# If there are multiple <link rel="icon">s, the browser uses their media,
|
|
# type, and sizes attributes to select the most appropriate icon.
|
|
# If several icons are equally appropriate, the last one is used.
|
|
# If the most appropriate icon is later found to be inappropriate,
|
|
# for example because it uses an unsupported format,
|
|
# the browser proceeds to the next-most appropriate, and so on.
|
|
# # DEBUG: /!\ firefox load all favicon ???
|
|
|
|
# iOS Safari 'apple-touch-icon'
|
|
# Safari pinned tabs 'mask-icon'
|
|
# Android Chrome 'manifest'
|
|
# Edge and IE 12:
|
|
# - <meta name="msapplication-TileColor" content="#aaaaaa"> <meta name="theme-color" content="#ffffff">
|
|
# - <meta name="msapplication-config" content="/icons/browserconfig.xml">
|
|
|
|
# desktop browser 'shortcut icon' (older browser), 'icon'
|
|
for favicon_tag in ['icon', 'shortcut icon']:
|
|
if soup.head:
|
|
for icon in soup.head.find_all('link', attrs={'rel': lambda x : x and x.lower() == favicon_tag, 'href': True}):
|
|
set_icons.add(icon)
|
|
|
|
# # TODO: handle base64 favicon
|
|
for tag in set_icons:
|
|
icon_url = tag.get('href')
|
|
if icon_url:
|
|
if icon_url.startswith('//'):
|
|
icon_url = icon_url.replace('//', '/')
|
|
if icon_url.startswith('data:'):
|
|
# # TODO: handle base64 favicon
|
|
pass
|
|
else:
|
|
icon_url = urljoin(url, icon_url)
|
|
icon_url = urlparse(icon_url, scheme=urlparse(url).scheme).geturl()
|
|
favicon_urls.add(icon_url)
|
|
return favicon_urls
|
|
|
|
|
|
# # # - - # # #
|
|
|
|
|
|
################################################################################
|
|
|
|
# # TODO: handle prefix cookies
|
|
# # TODO: fill empty fields
|
|
def create_cookie_crawler(cookie_dict, domain, crawler_type='web'):
|
|
# check cookie domain filed
|
|
if not 'domain' in cookie_dict:
|
|
cookie_dict['domain'] = f'.{domain}'
|
|
|
|
# tor browser: disable secure cookie
|
|
if crawler_type == 'onion':
|
|
cookie_dict['secure'] = False
|
|
|
|
# force cookie domain
|
|
# url = urlparse(browser_cookie['Host raw'])
|
|
# domain = url.netloc.split(':', 1)[0]
|
|
# cookie_dict['domain'] = '.{}'.format(domain)
|
|
|
|
# change expire date
|
|
cookie_dict['expires'] = (datetime.now() + timedelta(days=10)).strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
|
|
return cookie_dict
|
|
|
|
def load_crawler_cookies(cookiejar_uuid, domain, crawler_type='web'):
|
|
cookies = get_cookiejar_cookies_list(cookiejar_uuid)
|
|
all_cookies = []
|
|
for cookie_dict in cookies:
|
|
all_cookies.append(create_cookie_crawler(cookie_dict, domain, crawler_type=crawler_type))
|
|
return all_cookies
|
|
|
|
################################################################################
|
|
|
|
def get_all_cookiejar():
|
|
return r_serv_onion.smembers('cookiejar:all')
|
|
|
|
def get_global_cookiejar():
|
|
cookiejars = r_serv_onion.smembers('cookiejar:global')
|
|
if not cookiejars:
|
|
cookiejars = []
|
|
return cookiejars
|
|
|
|
def get_user_cookiejar(user_id):
|
|
cookiejars = r_serv_onion.smembers('cookiejar:user:{}'.format(user_id))
|
|
if not cookiejars:
|
|
cookiejars = []
|
|
return cookiejars
|
|
|
|
def exist_cookiejar(cookiejar_uuid):
|
|
return r_serv_onion.exists('cookiejar_metadata:{}'.format(cookiejar_uuid))
|
|
|
|
def _set_cookiejar_date(cookiejar_uuid, date):
|
|
r_serv_onion.hset(f'cookiejar_metadata:{cookiejar_uuid}', 'date', date)
|
|
|
|
# # TODO: sanitize cookie_uuid
|
|
def create_cookiejar(user_id, level=1, description=None, cookiejar_uuid=None):
|
|
if not cookiejar_uuid:
|
|
cookiejar_uuid = str(uuid.uuid4())
|
|
|
|
r_serv_onion.sadd('cookiejar:all', cookiejar_uuid)
|
|
if level == 0:
|
|
r_serv_onion.sadd(f'cookiejar:user:{user_id}', cookiejar_uuid)
|
|
else:
|
|
r_serv_onion.sadd('cookiejar:global', cookiejar_uuid)
|
|
# metadata
|
|
r_serv_onion.hset(f'cookiejar_metadata:{cookiejar_uuid}', 'user_id', user_id)
|
|
r_serv_onion.hset(f'cookiejar_metadata:{cookiejar_uuid}', 'level', level)
|
|
r_serv_onion.hset(f'cookiejar_metadata:{cookiejar_uuid}', 'description', description)
|
|
_set_cookiejar_date(cookiejar_uuid, datetime.now().strftime("%Y%m%d"))
|
|
|
|
# if json_cookies:
|
|
# json_cookies = json.loads(json_cookies) # # TODO: catch Exception
|
|
# r_serv_onion.set('cookies:json_cookies:{}'.format(cookies_uuid), json.dumps(json_cookies))
|
|
#
|
|
# for cookie_dict in l_cookies:
|
|
# r_serv_onion.hset('cookies:manual_cookies:{}'.format(cookies_uuid), cookie_dict['name'], cookie_dict['value'])
|
|
return cookiejar_uuid
|
|
|
|
def delete_cookie_jar(cookiejar_uuid):
|
|
level = get_cookiejar_level(cookiejar_uuid)
|
|
if level == 0:
|
|
user_id = get_cookiejar_owner(cookiejar_uuid)
|
|
r_serv_onion.srem('cookiejar:user:{}'.format(user_id), cookiejar_uuid)
|
|
else:
|
|
r_serv_onion.srem('cookiejar:global', cookiejar_uuid)
|
|
|
|
r_serv_onion.delete('cookiejar_metadata:{}'.format(cookiejar_uuid))
|
|
|
|
def get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
cookies = r_serv_onion.smembers('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid))
|
|
if not cookies:
|
|
cookies = []
|
|
return cookies
|
|
|
|
def get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=False):
|
|
l_cookiejar = []
|
|
for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
if add_cookie_uuid:
|
|
l_cookiejar.append((get_cookie_dict(cookie_uuid), cookie_uuid))
|
|
else:
|
|
l_cookiejar.append(get_cookie_dict(cookie_uuid))
|
|
return l_cookiejar
|
|
|
|
## Cookiejar metadata ##
|
|
def get_cookiejar_description(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description')
|
|
|
|
def get_cookiejar_date(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date')
|
|
|
|
def get_cookiejar_owner(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'user_id')
|
|
|
|
def get_cookiejar_date(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date')
|
|
|
|
def get_cookiejar_level(cookiejar_uuid):
|
|
level = r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'level')
|
|
if not level:
|
|
level = 1
|
|
return int(level)
|
|
|
|
def get_cookiejar_metadata(cookiejar_uuid, level=False):
|
|
dict_cookiejar = {}
|
|
if exist_cookiejar(cookiejar_uuid):
|
|
dict_cookiejar['cookiejar_uuid'] = cookiejar_uuid
|
|
dict_cookiejar['description'] = get_cookiejar_description(cookiejar_uuid)
|
|
dict_cookiejar['date'] = get_cookiejar_date(cookiejar_uuid)
|
|
dict_cookiejar['user_id'] = get_cookiejar_owner(cookiejar_uuid)
|
|
if level:
|
|
dict_cookiejar['level'] = get_cookiejar_level(cookiejar_uuid)
|
|
return dict_cookiejar
|
|
|
|
def get_cookiejar_metadata_by_iterator(iter_cookiejar_uuid):
|
|
l_cookiejar_metadata = []
|
|
for cookiejar_uuid in iter_cookiejar_uuid:
|
|
l_cookiejar_metadata.append(get_cookiejar_metadata(cookiejar_uuid))
|
|
return l_cookiejar_metadata
|
|
|
|
def edit_cookiejar_description(cookiejar_uuid, description):
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description', description)
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# COOKIES #
|
|
# #
|
|
# # # # # # # #
|
|
|
|
# # # #
|
|
# Cookies Fields:
|
|
# - name
|
|
# - value
|
|
# - path (optional)
|
|
# - domain (optional)
|
|
# - secure (optional)
|
|
# - httpOnly (optional)
|
|
# - text (optional)
|
|
# # # #
|
|
def get_cookie_all_keys_name():
|
|
return ['name', 'value', 'domain', 'path', 'httpOnly', 'secure']
|
|
|
|
def exists_cookie(cookie_uuid):
|
|
if int(r_serv_onion.scard(f'cookies:map:cookiejar:{cookie_uuid}')) > 0:
|
|
return True
|
|
return False
|
|
|
|
def get_cookie_value(cookie_uuid, name):
|
|
return r_serv_onion.hget(f'cookiejar:cookie:{cookie_uuid}', name)
|
|
|
|
def set_cookie_value(cookie_uuid, name, value):
|
|
r_serv_onion.hset(f'cookiejar:cookie:{cookie_uuid}', name, value)
|
|
|
|
def delete_cookie_value(cookie_uuid, name):
|
|
r_serv_onion.hdel(f'cookiejar:cookie:{cookie_uuid}', name)
|
|
|
|
def get_cookie_dict(cookie_uuid):
|
|
cookie_dict = {}
|
|
for key_name in r_serv_onion.hkeys(f'cookiejar:cookie:{cookie_uuid}'):
|
|
cookie_dict[key_name] = get_cookie_value(cookie_uuid, key_name)
|
|
return cookie_dict
|
|
|
|
# name, value, path=None, httpOnly=None, secure=None, domain=None, text=None
|
|
def add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict, cookie_uuid=None):
|
|
# # TODO: sanitize cookie_uuid
|
|
if not cookie_uuid:
|
|
cookie_uuid = generate_uuid()
|
|
r_serv_onion.sadd(f'cookiejar:{cookiejar_uuid}:cookies:uuid', cookie_uuid)
|
|
r_serv_onion.sadd(f'cookies:map:cookiejar:{cookie_uuid}', cookiejar_uuid)
|
|
|
|
set_cookie_value(cookie_uuid, 'name', cookie_dict['name'])
|
|
set_cookie_value(cookie_uuid, 'value', cookie_dict['value'])
|
|
if 'path' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'path', cookie_dict['path'])
|
|
if 'httpOnly' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'httpOnly', cookie_dict['httpOnly'])
|
|
if 'secure' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'secure', cookie_dict['secure'])
|
|
if 'domain' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'domain', cookie_dict['domain'])
|
|
if 'text' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'text', cookie_dict['text'])
|
|
return cookie_uuid
|
|
|
|
def add_cookies_to_cookiejar(cookiejar_uuid, l_cookies):
|
|
for cookie_dict in l_cookies:
|
|
add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
|
|
def delete_all_cookies_from_cookiejar(cookiejar_uuid):
|
|
for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid)
|
|
|
|
def delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid):
|
|
r_serv_onion.srem(f'cookiejar:{cookiejar_uuid}:cookies:uuid', cookie_uuid)
|
|
r_serv_onion.srem(f'cookies:map:cookiejar:{cookie_uuid}', cookiejar_uuid)
|
|
if not exists_cookie(cookie_uuid):
|
|
r_serv_onion.delete(f'cookiejar:cookie:{cookie_uuid}')
|
|
|
|
def edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict):
|
|
# delete old keys
|
|
for key_name in r_serv_onion.hkeys(f'cookiejar:cookie:{cookie_uuid}'):
|
|
if key_name not in cookie_dict:
|
|
delete_cookie_value(cookie_uuid, key_name)
|
|
# add new keys
|
|
cookie_all_keys_name = get_cookie_all_keys_name()
|
|
for key_name in cookie_dict:
|
|
if key_name in cookie_all_keys_name:
|
|
set_cookie_value(cookie_uuid, key_name, cookie_dict[key_name])
|
|
|
|
## - - ##
|
|
## Cookies import ## # TODO: add browser type ?
|
|
def import_cookies_from_json(json_cookies, cookiejar_uuid):
|
|
for cookie in json_cookies:
|
|
try:
|
|
cookie_dict = unpack_imported_json_cookie(cookie)
|
|
add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
except KeyError:
|
|
return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid}
|
|
|
|
# # TODO: add text field
|
|
def unpack_imported_json_cookie(json_cookie):
|
|
cookie_dict = {'name': json_cookie['Name raw'], 'value': json_cookie['Content raw']}
|
|
if 'Path raw' in json_cookie:
|
|
cookie_dict['path'] = json_cookie['Path raw']
|
|
if 'httpOnly' in json_cookie:
|
|
cookie_dict['httpOnly'] = json_cookie['HTTP only raw'] == 'true'
|
|
if 'secure' in json_cookie:
|
|
cookie_dict['secure'] = json_cookie['Send for'] == 'Encrypted connections only'
|
|
if 'Host raw' in json_cookie:
|
|
url = urlparse(json_cookie['Host raw'])
|
|
cookie_dict['domain'] = url.netloc.split(':', 1)[0]
|
|
return cookie_dict
|
|
|
|
def misp_cookie_import(misp_object, cookiejar_uuid):
|
|
pass
|
|
## - - ##
|
|
#### COOKIEJAR API ####
|
|
def api_import_cookies_from_json(json_cookies_str, cookiejar_uuid): # # TODO: add catch
|
|
json_cookies = json.loads(json_cookies_str)
|
|
resp = import_cookies_from_json(json_cookies, cookiejar_uuid)
|
|
if resp:
|
|
return resp, 400
|
|
#### ####
|
|
|
|
#### COOKIES API ####
|
|
|
|
def api_verify_basic_cookiejar(cookiejar_uuid, user_id):
|
|
if not exist_cookiejar(cookiejar_uuid):
|
|
return {'error': 'unknown cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404
|
|
level = get_cookiejar_level(cookiejar_uuid)
|
|
if level == 0: # # TODO: check if user is admin
|
|
cookie_owner = get_cookiejar_owner(cookiejar_uuid)
|
|
if cookie_owner != user_id:
|
|
return {'error': 'The access to this cookiejar is restricted'}, 403
|
|
|
|
def api_get_cookiejar_cookies(cookiejar_uuid, user_id):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
resp = get_cookiejar_cookies_list(cookiejar_uuid)
|
|
return resp, 200
|
|
|
|
def api_edit_cookiejar_description(user_id, cookiejar_uuid, description):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
edit_cookiejar_description(cookiejar_uuid, description)
|
|
return {'cookiejar_uuid': cookiejar_uuid}, 200
|
|
|
|
def api_get_cookiejar_cookies_with_uuid(cookiejar_uuid, user_id):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
resp = get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=True)
|
|
return resp, 200
|
|
|
|
def api_get_cookies_list_select(user_id):
|
|
l_cookiejar = []
|
|
for cookies_uuid in get_global_cookiejar():
|
|
l_cookiejar.append(f'{get_cookiejar_description(cookies_uuid)} : {cookies_uuid}')
|
|
for cookies_uuid in get_user_cookiejar(user_id):
|
|
l_cookiejar.append(f'{get_cookiejar_description(cookies_uuid)} : {cookies_uuid}')
|
|
return sorted(l_cookiejar)
|
|
|
|
def api_delete_cookie_from_cookiejar(user_id, cookiejar_uuid, cookie_uuid):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid)
|
|
return {'cookiejar_uuid': cookiejar_uuid, 'cookie_uuid': cookie_uuid}, 200
|
|
|
|
def api_delete_cookie_jar(user_id, cookiejar_uuid):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
delete_cookie_jar(cookiejar_uuid)
|
|
return {'cookiejar_uuid': cookiejar_uuid}, 200
|
|
|
|
def api_edit_cookie(user_id, cookiejar_uuid, cookie_uuid, cookie_dict):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '':
|
|
return {'error': 'cookie name or value not provided'}, 400
|
|
edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict)
|
|
return get_cookie_dict(cookie_uuid), 200
|
|
|
|
def api_create_cookie(user_id, cookiejar_uuid, cookie_dict):
|
|
resp = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if resp:
|
|
return resp
|
|
if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '':
|
|
return {'error': 'cookie name or value not provided'}, 400
|
|
resp = add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
return resp, 200
|
|
|
|
#### ####
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# CRAWLER # ###################################################################################
|
|
# #
|
|
# # # # # # # #
|
|
|
|
|
|
@unique
|
|
class CaptureStatus(IntEnum):
|
|
"""The status of the capture"""
|
|
UNKNOWN = -1
|
|
QUEUED = 0
|
|
DONE = 1
|
|
ONGOING = 2
|
|
|
|
def get_default_user_agent():
|
|
return 'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0'
|
|
|
|
def get_blacklist():
|
|
return r_crawler.smembers('blacklist:domain')
|
|
|
|
def is_blacklisted_domain(domain):
|
|
return r_crawler.sismember('blacklist:domain', domain)
|
|
|
|
def blacklist_domain(domain):
|
|
return r_crawler.sadd('blacklist:domain', domain)
|
|
|
|
def load_blacklist():
|
|
try:
|
|
with open(os.path.join(os.environ['AIL_BIN'], 'crawlers/blacklist.txt'), 'r') as f:
|
|
r_crawler.delete('blacklist:domain')
|
|
lines = f.read().splitlines()
|
|
for line in lines:
|
|
blacklist_domain(line)
|
|
# TODO LOG
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
def get_last_crawled_domains(domain_type):
|
|
return r_crawler.lrange(f'last_{domain_type}', 0, -1)
|
|
|
|
def update_last_crawled_domain(domain_type, domain, epoch):
|
|
# update list, last crawled domains
|
|
r_crawler.lpush(f'last_{domain_type}', f'{domain}:{epoch}')
|
|
r_crawler.ltrim(f'last_{domain_type}', 0, 15)
|
|
|
|
def create_item_metadata(item_id, url, item_father):
|
|
item = Item(item_id)
|
|
item.set_crawled(url, item_father)
|
|
|
|
def get_gzipped_b64_item(item_id, content):
|
|
try:
|
|
gzipencoded = gzip.compress(content.encode())
|
|
gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
|
|
return gzip64encoded
|
|
except:
|
|
print(f'file error: {item_id}')
|
|
return False
|
|
|
|
def get_crawlers_stats_by_day(date, domain_type):
|
|
return {
|
|
'date': date[0:4] + '-' + date[4:6] + '-' + date[6:8],
|
|
'up': r_crawler.scard(f'{domain_type}_up:{date}'),
|
|
'down': r_crawler.scard(f'{domain_type}_down:{date}'),
|
|
}
|
|
|
|
|
|
def get_crawlers_stats(domain_type=None):
|
|
stats = {}
|
|
date = datetime.now().strftime("%Y%m%d")
|
|
if domain_type:
|
|
domain_types = [domain_type]
|
|
else:
|
|
domain_types = get_crawler_all_types()
|
|
for domain_type in domain_types:
|
|
queue = r_crawler.scard(f'crawler:queue:type:{domain_type}')
|
|
up = r_crawler.scard(f'{domain_type}_up:{date}')
|
|
down = r_crawler.scard(f'{domain_type}_down:{date}')
|
|
crawled = up + down
|
|
stats[domain_type] = {'queue': queue, 'up': up, 'down': down, 'crawled': crawled}
|
|
return stats
|
|
|
|
#### CRAWLER STATE ####
|
|
|
|
# TODO SET IN UI OR USE DEFAULT
|
|
def get_crawler_max_captures():
|
|
return 10
|
|
|
|
def get_nb_crawler_captures():
|
|
return r_cache.zcard('crawler:captures')
|
|
|
|
def get_crawler_captures():
|
|
return r_crawler.zrange('crawler:captures', 0, -1)
|
|
|
|
def reload_crawler_captures():
|
|
r_cache.delete('crawler:captures')
|
|
for capture in get_crawler_captures():
|
|
r_cache.zadd('crawler:captures', {capture[0]: capture[1]})
|
|
|
|
def get_crawler_capture():
|
|
return r_cache.zpopmin('crawler:captures')
|
|
|
|
def update_crawler_capture(capture_uuid):
|
|
last_check = int(time.time())
|
|
r_cache.zadd('crawler:captures', {capture_uuid: last_check})
|
|
|
|
def get_crawler_capture_task_uuid(capture_uuid):
|
|
return r_crawler.hget('crawler:captures:tasks', capture_uuid)
|
|
|
|
def add_crawler_capture(task_uuid, capture_uuid):
|
|
launch_time = int(time.time())
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'capture', capture_uuid)
|
|
r_crawler.hset('crawler:captures:tasks', capture_uuid, task_uuid)
|
|
r_crawler.zadd('crawler:captures', {capture_uuid: launch_time})
|
|
r_cache.zadd('crawler:captures', {capture_uuid: launch_time})
|
|
|
|
def remove_crawler_capture(capture_uuid):
|
|
r_crawler.zrem('crawler:captures', capture_uuid)
|
|
r_crawler.hdel('crawler:captures:tasks', capture_uuid)
|
|
|
|
def get_crawler_capture_status():
|
|
status = []
|
|
for capture_uuid in get_crawler_captures():
|
|
task_uuid = get_crawler_capture_task_uuid(capture_uuid)
|
|
domain = get_crawler_task_domain(task_uuid)
|
|
dom = Domain(domain)
|
|
meta = {
|
|
'uuid': task_uuid,
|
|
'domain': dom.get_id(),
|
|
'type': dom.get_domain_type(),
|
|
'start_time': get_crawler_task_start_time(task_uuid),
|
|
'status': 'test',
|
|
}
|
|
status.append(meta)
|
|
return status
|
|
|
|
##-- CRAWLER STATE --##
|
|
|
|
#### CRAWLER TASK ####
|
|
|
|
def get_crawler_task_url(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'url')
|
|
|
|
def get_crawler_task_domain(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'domain')
|
|
|
|
def get_crawler_task_depth(task_uuid):
|
|
depth = r_crawler.hget(f'crawler:task:{task_uuid}', 'depth')
|
|
if not depth:
|
|
depth = 1
|
|
return int(depth)
|
|
|
|
def get_crawler_task_har(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'har') == '1'
|
|
|
|
def get_crawler_task_screenshot(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'screenshot') == '1'
|
|
|
|
def get_crawler_task_user_agent(task_uuid):
|
|
user_agent = r_crawler.hget(f'crawler:task:{task_uuid}', 'user_agent')
|
|
if not user_agent:
|
|
user_agent = get_default_user_agent()
|
|
return user_agent
|
|
|
|
def get_crawler_task_cookiejar(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'cookiejar')
|
|
|
|
def get_crawler_task_header(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'header')
|
|
|
|
def get_crawler_task_proxy(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'proxy')
|
|
|
|
def get_crawler_task_parent(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'parent')
|
|
|
|
def get_crawler_task_hash(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'hash')
|
|
|
|
def get_crawler_task_start_time(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'start_time')
|
|
|
|
def get_crawler_task_status(task_uuid):
|
|
return r_crawler.hget(f'crawler:task:{task_uuid}', 'status')
|
|
|
|
def get_crawler_task_capture(task_uuid):
|
|
return r_crawler.hset(f'crawler:task:{task_uuid}', 'capture')
|
|
|
|
def get_crawler_task(task_uuid):
|
|
meta = {
|
|
'uuid': task_uuid,
|
|
'url': get_crawler_task_url(task_uuid),
|
|
'domain': get_crawler_task_domain(task_uuid),
|
|
'depth': get_crawler_task_depth(task_uuid),
|
|
'har': get_crawler_task_har(task_uuid),
|
|
'screenshot': get_crawler_task_screenshot(task_uuid),
|
|
'user_agent': get_crawler_task_user_agent(task_uuid),
|
|
'cookiejar': get_crawler_task_cookiejar(task_uuid),
|
|
'header': get_crawler_task_header(task_uuid),
|
|
'proxy': get_crawler_task_proxy(task_uuid),
|
|
'parent': get_crawler_task_parent(task_uuid),
|
|
}
|
|
return meta
|
|
|
|
def get_task_status(task_uuid):
|
|
domain = get_crawler_task_domain(task_uuid)
|
|
dom = Domain(domain)
|
|
meta = {
|
|
'uuid': task_uuid,
|
|
'domain': dom.get_id(),
|
|
'domain_type': dom.get_domain_type(),
|
|
'start_time': get_crawler_task_start_time(task_uuid),
|
|
'status': 'test',
|
|
}
|
|
return meta
|
|
|
|
# domain -> uuid
|
|
def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header):
|
|
to_enqueue = {'domain': domain, 'depth': depth, 'har': har, 'screenshot': screenshot,
|
|
'priority': priority, 'proxy': proxy, 'cookiejar': cookiejar, 'user_agent': user_agent,
|
|
'header': header}
|
|
if priority != 0:
|
|
to_enqueue['url'] = url
|
|
return hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest()
|
|
|
|
# TODO STATUS UPDATE
|
|
# PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100
|
|
def add_crawler_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, parent='manual', priority=0):
|
|
url_decoded = unpack_url(url)
|
|
url = url_decoded['url']
|
|
domain = url_decoded['domain']
|
|
dom = Domain(domain)
|
|
|
|
# Discovery crawler
|
|
if priority == 0:
|
|
if is_blacklisted_domain(dom.get_id()):
|
|
return None
|
|
if not dom.exists():
|
|
priority = 10
|
|
# Domain Crawled today or UP this month
|
|
if dom.is_down_today() or dom.is_up_this_month():
|
|
return None
|
|
|
|
har = int(har)
|
|
screenshot = int(screenshot)
|
|
|
|
# TODO SELECT PROXY -> URL TODO SELECT PROXY
|
|
if proxy == 'web':
|
|
proxy = None
|
|
else:
|
|
proxy = 'force_tor'
|
|
if not user_agent:
|
|
user_agent = get_default_user_agent()
|
|
|
|
# TODO COOKIEJAR -> UUID
|
|
if cookiejar:
|
|
pass
|
|
|
|
# Check if already in queue
|
|
hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header)
|
|
if r_crawler.hexists(f'crawler:queue:hash', hash_query):
|
|
return r_crawler.hget(f'crawler:queue:hash', hash_query)
|
|
|
|
# TODO ADD TASK STATUS -----
|
|
task_uuid = gen_uuid() # TODO Save hash ??? (just to be safe and remove it)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'domain', domain)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'url', url)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'depth', int(depth))
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'har', har)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'screenshot', har)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'user_agent', user_agent)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'proxy', proxy)
|
|
if cookiejar:
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'cookiejar', cookiejar) # TODO
|
|
if header:
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'header', header)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'hash', hash_query)
|
|
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'parent', parent)
|
|
|
|
r_crawler.hset('crawler:queue:hash', hash_query, task_uuid)
|
|
r_crawler.zadd('crawler:queue', {task_uuid: priority})
|
|
# UI
|
|
r_crawler.sadd(f'crawler:queue:type:{dom.get_domain_type()}', task_uuid)
|
|
return task_uuid
|
|
|
|
def get_crawler_task_from_queue():
|
|
task_uuid = r_crawler.zpopmax('crawler:queue')
|
|
if not task_uuid or not task_uuid[0]:
|
|
return None
|
|
task_uuid, priority = task_uuid[0]
|
|
r_crawler.sadd('crawler:queue:queued', task_uuid)
|
|
r_crawler.hset(f'crawler:task:{task_uuid}', 'start_time', datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
|
|
return task_uuid, priority
|
|
|
|
def clear_crawler_task(task_uuid, domain_type):
|
|
hash_query = get_crawler_task_hash(task_uuid)
|
|
r_crawler.hdel('crawler:queue:hash', hash_query)
|
|
r_crawler.srem(f'crawler:queue:type:{domain_type}', task_uuid)
|
|
r_crawler.srem('crawler:queue:queued', task_uuid)
|
|
|
|
def get_crawlers_tasks_status():
|
|
tasks_status = []
|
|
tasks = r_crawler.smembers('crawler:queue:queued')
|
|
for task_uuid in tasks:
|
|
tasks_status.append(get_task_status(task_uuid))
|
|
return tasks_status
|
|
|
|
##-- CRAWLER TASK --##
|
|
|
|
#### CRAWLER TASK API ####
|
|
|
|
# # TODO: ADD user agent
|
|
# # TODO: sanitize URL
|
|
def api_add_crawler_task(data, user_id=None):
|
|
url = data.get('url', None)
|
|
if not url or url=='\n':
|
|
return ({'status': 'error', 'reason': 'No url supplied'}, 400)
|
|
|
|
screenshot = data.get('screenshot', False)
|
|
if screenshot:
|
|
screenshot = True
|
|
else:
|
|
screenshot = False
|
|
har = data.get('har', False)
|
|
if har:
|
|
har = True
|
|
else:
|
|
har = False
|
|
depth_limit = data.get('depth_limit', 1)
|
|
if depth_limit:
|
|
try:
|
|
depth_limit = int(depth_limit)
|
|
if depth_limit < 0:
|
|
depth_limit = 0
|
|
except ValueError:
|
|
return ({'error':'invalid depth limit'}, 400)
|
|
else:
|
|
depth_limit = 0
|
|
|
|
# cookiejar_uuid = data.get('cookiejar_uuid', None)
|
|
# if cookiejar_uuid:
|
|
# if not exist_cookiejar(cookiejar_uuid):
|
|
# return ({'error': 'unknow cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404)
|
|
# level = get_cookiejar_level(cookiejar_uuid)
|
|
# if level == 0: # # TODO: check if user is admin ######################################################
|
|
# cookie_owner = get_cookiejar_owner(cookiejar_uuid)
|
|
# if cookie_owner != user_id:
|
|
# return ({'error': 'The access to this cookiejar is restricted'}, 403)
|
|
|
|
# if auto_crawler:
|
|
# try:
|
|
# crawler_delta = int(crawler_delta)
|
|
# if crawler_delta < 0:
|
|
# return ({'error':'invalid delta between two pass of the crawler'}, 400)
|
|
# except ValueError:
|
|
# return ({'error':'invalid delta between two pass of the crawler'}, 400)
|
|
|
|
proxy = data.get('proxy', None)
|
|
if proxy == 'onion' or proxy == 'tor':
|
|
proxy = 'force_tor'
|
|
else:
|
|
# TODO sanitize PROXY
|
|
proxy = None
|
|
|
|
# TODO #############################################################################################################
|
|
auto_crawler = auto_crawler
|
|
crawler_delta = crawler_delta
|
|
parent = 'manual'
|
|
|
|
return add_crawler_task(url, depth=depth_limit, har=har, screenshot=screenshot, cookiejar=cookiejar_uuid,
|
|
proxy=proxy, user_agent=user_agent, parent='manual', priority=90), 200
|
|
|
|
|
|
|
|
|
|
|
|
#### ####
|
|
|
|
|
|
###################################################################################
|
|
###################################################################################
|
|
###################################################################################
|
|
###################################################################################
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#### CRAWLER GLOBAL ####
|
|
|
|
# TODO: # FIXME: config db, dynamic load
|
|
def is_crawler_activated():
|
|
return activate_crawler == 'True'
|
|
|
|
def get_crawler_all_types():
|
|
return ['onion', 'web']
|
|
|
|
def sanitize_crawler_types(l_crawler_types):
|
|
all_crawler_types = get_crawler_all_types()
|
|
if not l_crawler_types:
|
|
return all_crawler_types
|
|
for crawler_type in l_crawler_types:
|
|
if crawler_type not in all_crawler_types:
|
|
return all_crawler_types
|
|
return l_crawler_types
|
|
|
|
def get_all_spash_crawler_status():
|
|
crawler_metadata = []
|
|
all_crawlers = r_cache.smembers('all_splash_crawlers')
|
|
for crawler in all_crawlers:
|
|
crawler_metadata.append(get_splash_crawler_status(crawler))
|
|
return crawler_metadata
|
|
|
|
def reset_all_spash_crawler_status():
|
|
r_cache.delete('all_splash_crawlers')
|
|
|
|
def get_splash_crawler_status(spash_url):
|
|
crawler_type = r_cache.hget(f'metadata_crawler:{spash_url}', 'type')
|
|
crawling_domain = r_cache.hget(f'metadata_crawler:{spash_url}', 'crawling_domain')
|
|
started_time = r_cache.hget(f'metadata_crawler:{spash_url}', 'started_time')
|
|
status_info = r_cache.hget(f'metadata_crawler:{spash_url}', 'status')
|
|
crawler_info = f'{spash_url} - {started_time}'
|
|
if status_info=='Waiting' or status_info=='Crawling':
|
|
status=True
|
|
else:
|
|
status=False
|
|
return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type}
|
|
|
|
def set_current_crawler_status(splash_url, status, started_time=False, crawled_domain=None, crawler_type=None):
|
|
# TODO: get crawler type if None
|
|
# Status: ['Waiting', 'Error', ...]
|
|
r_cache.hset(f'metadata_crawler:{splash_url}', 'status', status)
|
|
if started_time:
|
|
r_cache.hset(f'metadata_crawler:{splash_url}', 'started_time', datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
|
|
if crawler_type:
|
|
r_cache.hset(f'metadata_crawler:{splash_url}', 'type', crawler_type)
|
|
if crawled_domain:
|
|
r_cache.hset(f'metadata_crawler:{splash_url}', 'crawling_domain', crawled_domain)
|
|
|
|
#r_cache.sadd('all_splash_crawlers', splash_url) # # TODO: add me in fct: create_ail_crawler
|
|
|
|
def get_stats_last_crawled_domains(crawler_types, date):
|
|
statDomains = {}
|
|
for crawler_type in crawler_types:
|
|
stat_type = {}
|
|
stat_type['domains_up'] = r_crawler.scard('{}_up:{}'.format(crawler_type, date))
|
|
stat_type['domains_down'] = r_crawler.scard('{}_down:{}'.format(crawler_type, date))
|
|
stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down']
|
|
stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type)
|
|
statDomains[crawler_type] = stat_type
|
|
return statDomains
|
|
|
|
# # TODO: handle custom proxy
|
|
def get_splash_crawler_latest_stats():
|
|
now = datetime.now()
|
|
date = now.strftime("%Y%m%d")
|
|
return get_stats_last_crawled_domains(['onion', 'web'], date)
|
|
|
|
def get_nb_crawlers_to_launch_by_splash_name(splash_name):
|
|
res = r_serv_onion.hget('all_crawlers_to_launch', splash_name)
|
|
if res:
|
|
return int(res)
|
|
else:
|
|
return 0
|
|
|
|
def get_all_crawlers_to_launch_splash_name():
|
|
return r_serv_onion.hkeys('all_crawlers_to_launch')
|
|
|
|
def get_nb_crawlers_to_launch():
|
|
nb_crawlers_to_launch = r_serv_onion.hgetall('all_crawlers_to_launch')
|
|
for splash_name in nb_crawlers_to_launch:
|
|
nb_crawlers_to_launch[splash_name] = int(nb_crawlers_to_launch[splash_name])
|
|
return nb_crawlers_to_launch
|
|
|
|
def get_nb_crawlers_to_launch_ui():
|
|
nb_crawlers_to_launch = get_nb_crawlers_to_launch()
|
|
for splash_name in get_all_splash():
|
|
if splash_name not in nb_crawlers_to_launch:
|
|
nb_crawlers_to_launch[splash_name] = 0
|
|
return nb_crawlers_to_launch
|
|
|
|
def set_nb_crawlers_to_launch(dict_splash_name):
|
|
r_serv_onion.delete('all_crawlers_to_launch')
|
|
for splash_name in dict_splash_name:
|
|
r_serv_onion.hset('all_crawlers_to_launch', splash_name, int(dict_splash_name[splash_name]))
|
|
relaunch_crawlers()
|
|
|
|
def relaunch_crawlers():
|
|
all_crawlers_to_launch = get_nb_crawlers_to_launch()
|
|
for splash_name in all_crawlers_to_launch:
|
|
nb_crawlers = int(all_crawlers_to_launch[splash_name])
|
|
|
|
all_crawler_urls = get_splash_all_url(splash_name, r_list=True)
|
|
if nb_crawlers > len(all_crawler_urls):
|
|
print('Error, can\'t launch all Splash Dockers')
|
|
print('Please launch {} additional {} Dockers'.format( nb_crawlers - len(all_crawler_urls), splash_name))
|
|
nb_crawlers = len(all_crawler_urls)
|
|
|
|
reset_all_spash_crawler_status()
|
|
|
|
for i in range(0, int(nb_crawlers)):
|
|
splash_url = all_crawler_urls[i]
|
|
print(all_crawler_urls[i])
|
|
|
|
launch_ail_splash_crawler(splash_url, script_options='{}'.format(splash_url))
|
|
|
|
def api_set_nb_crawlers_to_launch(dict_splash_name):
|
|
# TODO: check if is dict
|
|
dict_crawlers_to_launch = {}
|
|
all_splash = get_all_splash()
|
|
crawlers_to_launch = list(set(all_splash) & set(dict_splash_name.keys()))
|
|
for splash_name in crawlers_to_launch:
|
|
try:
|
|
nb_to_launch = int(dict_splash_name.get(splash_name, 0))
|
|
if nb_to_launch < 0:
|
|
return {'error':'The number of crawlers to launch is negative'}, 400
|
|
except:
|
|
return {'error':'invalid number of crawlers to launch'}, 400
|
|
if nb_to_launch > 0:
|
|
dict_crawlers_to_launch[splash_name] = nb_to_launch
|
|
|
|
if dict_crawlers_to_launch:
|
|
set_nb_crawlers_to_launch(dict_crawlers_to_launch)
|
|
return (dict_crawlers_to_launch, 200)
|
|
else:
|
|
return {'error':'invalid input'}, 400
|
|
|
|
def add_domain_blacklist(domain_type, domain):
|
|
r_serv_onion.sadd(f'blacklist_{domain_type}', domain)
|
|
|
|
##-- CRAWLER GLOBAL --##
|
|
|
|
#### AUTOMATIC CRAWLER ####
|
|
|
|
def get_auto_crawler_all_domain(l_crawler_types=[]):
|
|
l_crawler_types = sanitize_crawler_types(l_crawler_types)
|
|
if len(l_crawler_types) == 1:
|
|
return r_serv_onion.smembers(f'auto_crawler_url:{l_crawler_types[0]}')
|
|
else:
|
|
l_keys_name = []
|
|
for crawler_type in l_crawler_types:
|
|
l_keys_name.append(f'auto_crawler_url:{crawler_type}')
|
|
return r_serv_onion.sunion(l_keys_name[0], *l_keys_name[1:])
|
|
|
|
def add_auto_crawler_in_queue(domain, domain_type, port, epoch, delta, message):
|
|
r_serv_onion.zadd('crawler_auto_queue', {f'{message};{domain_type}': int(time.time() + delta)})
|
|
# update list, last auto crawled domains
|
|
r_serv_onion.lpush('last_auto_crawled', f'{domain}:{port};{epoch}')
|
|
r_serv_onion.ltrim('last_auto_crawled', 0, 9)
|
|
|
|
def update_auto_crawler_queue():
|
|
current_epoch = int(time.time())
|
|
# check if current_epoch > domain_next_epoch
|
|
l_queue = r_serv_onion.zrangebyscore('crawler_auto_queue', 0, current_epoch)
|
|
for elem in l_queue:
|
|
mess, domain_type = elem.rsplit(';', 1)
|
|
print(domain_type)
|
|
print(mess)
|
|
r_serv_onion.sadd(f'{domain_type}_crawler_priority_queue', mess)
|
|
|
|
|
|
##-- AUTOMATIC CRAWLER --##
|
|
|
|
#### CRAWLER TASK ####
|
|
|
|
|
|
|
|
##-- CRAWLER TASK --##
|
|
|
|
|
|
|
|
#### ####
|
|
|
|
|
|
def is_redirection(domain, last_url):
|
|
url = urlparse(last_url)
|
|
last_domain = url.netloc
|
|
last_domain = last_domain.split('.')
|
|
last_domain = '{}.{}'.format(last_domain[-2], last_domain[-1])
|
|
return domain != last_domain
|
|
|
|
def create_item_id(item_dir, domain):
|
|
# remove /
|
|
domain = domain.replace('/', '_')
|
|
if len(domain) > 215:
|
|
UUID = domain[-215:]+str(uuid.uuid4())
|
|
else:
|
|
UUID = domain+str(uuid.uuid4())
|
|
return os.path.join(item_dir, UUID)
|
|
|
|
def save_har(har_dir, item_id, har_content):
|
|
if not os.path.exists(har_dir):
|
|
os.makedirs(har_dir)
|
|
item_id = item_id.split('/')[-1]
|
|
filename = os.path.join(har_dir, item_id + '.json')
|
|
with open(filename, 'w') as f:
|
|
f.write(json.dumps(har_content))
|
|
|
|
#### CRAWLER QUEUES ####
|
|
|
|
## queues priority:
|
|
# 1 - priority queue
|
|
# 2 - discovery queue
|
|
# 3 - default queue
|
|
##
|
|
def get_all_queues_names():
|
|
return ['priority', 'discovery', 'default']
|
|
|
|
def get_all_queues_keys():
|
|
return ['{}_crawler_priority_queue', '{}_crawler_discovery_queue', '{}_crawler_queue']
|
|
|
|
def get_queue_key_by_name(queue_name):
|
|
if queue_name == 'priority':
|
|
return '{}_crawler_priority_queue'
|
|
elif queue_name == 'discovery':
|
|
return '{}_crawler_discovery_queue'
|
|
else: # default
|
|
return '{}_crawler_queue'
|
|
|
|
def get_stats_elem_to_crawl_by_queue_type(queue_type):
|
|
dict_stats = {}
|
|
for queue_name in get_all_queues_names():
|
|
dict_stats[queue_name] = r_serv_onion.scard(get_queue_key_by_name(queue_name).format(queue_type))
|
|
return dict_stats
|
|
|
|
def get_all_queues_stats():
|
|
dict_stats = {}
|
|
for queue_type in get_crawler_all_types():
|
|
dict_stats[queue_type] = get_stats_elem_to_crawl_by_queue_type(queue_type)
|
|
for queue_type in get_all_splash():
|
|
dict_stats[queue_type] = get_stats_elem_to_crawl_by_queue_type(queue_type)
|
|
return dict_stats
|
|
|
|
def is_domain_in_queue(queue_type, domain):
|
|
return r_serv_onion.sismember(f'{queue_type}_domain_crawler_queue', domain)
|
|
|
|
def is_item_in_queue(queue_type, url, item_id, queue_name=None):
|
|
if queue_name is None:
|
|
queues = get_all_queues_keys()
|
|
else:
|
|
queues = get_queue_key_by_name(queue_name)
|
|
|
|
key = f'{url};{item_id}'
|
|
for queue in queues:
|
|
if r_serv_onion.sismember(queue.format(queue_type), key):
|
|
return True
|
|
return False
|
|
|
|
def queue_test_clean_up(queue_type, domain, item_id):
|
|
date_month = datetime.now().strftime("%Y%m")
|
|
r_serv_onion.srem(f'month_{queue_type}_up:{date_month}', domain)
|
|
|
|
# Clean up
|
|
r_serv_onion.srem(f'{queue_type}_domain_crawler_queue', domain)
|
|
msg = f'{domain};{item_id}'
|
|
r_serv_onion.srem(f'{queue_type}_crawler_discovery_queue', msg)
|
|
r_serv_onion.srem(f'{queue_type}_crawler_queue', msg)
|
|
|
|
|
|
def remove_task_from_crawler_queue(queue_name, queue_type, key_to_remove):
|
|
r_serv_onion.srem(queue_name.format(queue_type), key_to_remove)
|
|
|
|
# # TODO: keep auto crawler ?
|
|
def clear_crawler_queues():
|
|
for queue_key in get_all_queues_keys():
|
|
for queue_type in get_crawler_all_types():
|
|
r_serv_onion.delete(queue_key.format(queue_type))
|
|
|
|
###################################################################################
|
|
def get_nb_elem_to_crawl_by_type(queue_type): # # TODO: rename me
|
|
nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type))
|
|
nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type))
|
|
nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type))
|
|
return nb
|
|
###################################################################################
|
|
|
|
def get_all_crawlers_queues_types():
|
|
all_queues_types = set()
|
|
all_splash_name = get_all_crawlers_to_launch_splash_name()
|
|
for splash_name in all_splash_name:
|
|
all_queues_types.add(get_splash_crawler_type(splash_name))
|
|
all_splash_name = list()
|
|
return all_queues_types
|
|
|
|
def get_crawler_queue_types_by_splash_name(splash_name):
|
|
all_domain_type = [splash_name]
|
|
crawler_type = get_splash_crawler_type(splash_name)
|
|
#if not is_splash_used_in_discovery(splash_name)
|
|
if crawler_type == 'tor':
|
|
all_domain_type.append('onion')
|
|
all_domain_type.append('web')
|
|
else:
|
|
all_domain_type.append('web')
|
|
return all_domain_type
|
|
|
|
def get_crawler_type_by_url(url):
|
|
faup.decode(url)
|
|
unpack_url = faup.get()
|
|
## TODO: # FIXME: remove me
|
|
try:
|
|
tld = unpack_url['tld'].decode()
|
|
except:
|
|
tld = unpack_url['tld']
|
|
|
|
if tld == 'onion':
|
|
crawler_type = 'onion'
|
|
else:
|
|
crawler_type = 'web'
|
|
return crawler_type
|
|
|
|
|
|
def get_elem_to_crawl_by_queue_type(l_queue_type):
|
|
## queues priority:
|
|
# 1 - priority queue
|
|
# 2 - discovery queue
|
|
# 3 - normal queue
|
|
##
|
|
|
|
for queue_key in get_all_queues_keys():
|
|
for queue_type in l_queue_type:
|
|
message = r_serv_onion.spop(queue_key.format(queue_type))
|
|
if message:
|
|
dict_to_crawl = {}
|
|
splitted = message.rsplit(';', 1)
|
|
if len(splitted) == 2:
|
|
url, item_id = splitted
|
|
item_id = item_id.replace(ITEMS_FOLDER+'/', '')
|
|
else:
|
|
# # TODO: to check/refractor
|
|
item_id = None
|
|
url = message
|
|
crawler_type = get_crawler_type_by_url(url)
|
|
return {'url': url, 'paste': item_id, 'type_service': crawler_type, 'queue_type': queue_type, 'original_message': message}
|
|
return None
|
|
|
|
#### ---- ####
|
|
|
|
# # # # # # # # # # # #
|
|
# #
|
|
# SPLASH MANAGER #
|
|
# #
|
|
# # # # # # # # # # # #
|
|
|
|
## PROXY ##
|
|
def get_all_proxies(r_list=False):
|
|
res = r_serv_onion.smembers('all_proxies')
|
|
if res:
|
|
return list(res)
|
|
else:
|
|
return []
|
|
|
|
def delete_all_proxies():
|
|
for proxy_name in get_all_proxies():
|
|
delete_proxy(proxy_name)
|
|
|
|
def get_proxy_host(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'host')
|
|
|
|
def get_proxy_port(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'port')
|
|
|
|
def get_proxy_type(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'type')
|
|
|
|
def get_proxy_crawler_type(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'crawler_type')
|
|
|
|
def get_proxy_description(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'description')
|
|
|
|
def get_proxy_metadata(proxy_name):
|
|
meta_dict = {}
|
|
meta_dict['host'] = get_proxy_host(proxy_name)
|
|
meta_dict['port'] = get_proxy_port(proxy_name)
|
|
meta_dict['type'] = get_proxy_type(proxy_name)
|
|
meta_dict['crawler_type'] = get_proxy_crawler_type(proxy_name)
|
|
meta_dict['description'] = get_proxy_description(proxy_name)
|
|
return meta_dict
|
|
|
|
def get_all_proxies_metadata():
|
|
all_proxy_dict = {}
|
|
for proxy_name in get_all_proxies():
|
|
all_proxy_dict[proxy_name] = get_proxy_metadata(proxy_name)
|
|
return all_proxy_dict
|
|
|
|
# def set_proxy_used_in_discovery(proxy_name, value):
|
|
# r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value)
|
|
|
|
def delete_proxy(proxy_name): # # TODO: force delete (delete all proxy)
|
|
proxy_splash = get_all_splash_by_proxy(proxy_name)
|
|
#if proxy_splash:
|
|
# print('error, a splash container is using this proxy')
|
|
r_serv_onion.delete('proxy:metadata:{}'.format(proxy_name))
|
|
r_serv_onion.srem('all_proxies', proxy_name)
|
|
## -- ##
|
|
|
|
#### ---- ####
|
|
|
|
|
|
# # # # CRAWLER LACUS # # # #
|
|
|
|
def get_lacus_url():
|
|
return r_db.hget('crawler:lacus', 'url')
|
|
|
|
def get_lacus_api_key(reload=False): # TODO: add in db config
|
|
return r_db.hget('crawler:lacus', 'key')
|
|
|
|
# TODO Rewrite with new API key
|
|
def get_hidden_lacus_api_key(): # TODO: add in db config
|
|
key = get_lacus_api_key()
|
|
if key:
|
|
if len(key)==41:
|
|
return f'{key[:4]}*********************************{key[-4:]}'
|
|
|
|
# TODO Rewrite with new API key
|
|
def is_valid_api_key(api_key, search=re.compile(r'[^a-zA-Z0-9_-]').search):
|
|
if len(api_key) != 41:
|
|
return False
|
|
return not bool(search(api_key))
|
|
|
|
def save_lacus_url_api(url, api_key):
|
|
r_db.hset('crawler:lacus', 'url', url)
|
|
r_db.hset('crawler:lacus', 'key', api_key)
|
|
|
|
def is_lacus_connected(delta_check=30):
|
|
last_check = r_cache.hget('crawler:lacus', 'last_check')
|
|
if last_check:
|
|
if int(time.time()) - int(last_check) > delta_check:
|
|
ping_lacus()
|
|
else:
|
|
ping_lacus()
|
|
is_connected = r_cache.hget('crawler:lacus', 'connected')
|
|
return is_connected == 'True'
|
|
|
|
def get_lacus_connection_metadata(force_ping=False):
|
|
dict_manager={}
|
|
if force_ping:
|
|
dict_manager['status'] = ping_lacus()
|
|
else:
|
|
dict_manager['status'] = is_lacus_connected()
|
|
if not dict_manager['status']:
|
|
dict_manager['status_code'] = r_cache.hget('crawler:lacus', 'status_code')
|
|
dict_manager['error'] = r_cache.hget('crawler:lacus', 'error')
|
|
return dict_manager
|
|
|
|
def get_lacus():
|
|
url = get_lacus_url()
|
|
if url:
|
|
return PyLacus(get_lacus_url())
|
|
|
|
# TODO CATCH EXCEPTIONS
|
|
def ping_lacus():
|
|
# TODO CATCH EXCEPTION
|
|
lacus = get_lacus()
|
|
if not lacus:
|
|
ping = False
|
|
else:
|
|
ping = lacus.is_up
|
|
update_lacus_connection_status(ping)
|
|
return ping
|
|
|
|
def update_lacus_connection_status(is_connected, req_error=None):
|
|
r_cache.hset('crawler:lacus', 'connected', str(is_connected))
|
|
r_cache.hset('crawler:lacus', 'last_check', int(time.time()))
|
|
if not req_error:
|
|
r_cache.hdel('crawler:lacus', 'error')
|
|
else:
|
|
r_cache.hset('crawler:lacus', 'status_code', req_error['status_code'])
|
|
r_cache.hset('crawler:lacus', 'error', req_error['error'])
|
|
|
|
def api_save_lacus_url_key(data):
|
|
# unpack json
|
|
manager_url = data.get('url', None)
|
|
api_key = data.get('api_key', None)
|
|
if not manager_url: # or not api_key:
|
|
return {'status': 'error', 'reason': 'No url or API key supplied'}, 400
|
|
# check if is valid url
|
|
try:
|
|
result = urlparse(manager_url)
|
|
if not all([result.scheme, result.netloc]):
|
|
return {'status': 'error', 'reason': 'Invalid url'}, 400
|
|
except:
|
|
return {'status': 'error', 'reason': 'Invalid url'}, 400
|
|
|
|
# # check if is valid key CURRENTLY DISABLE
|
|
# if not is_valid_api_key(api_key):
|
|
# return ({'status': 'error', 'reason': 'Invalid API key'}, 400)
|
|
|
|
save_lacus_url_api(manager_url, api_key)
|
|
return {'url': manager_url, 'api_key': get_hidden_lacus_api_key()}, 200
|
|
|
|
|
|
|
|
|
|
## PROXY ##
|
|
|
|
# TODO SAVE PROXY URL + ADD PROXY TESTS
|
|
# -> name + url
|
|
|
|
## PROXY ##
|
|
|
|
def is_test_ail_crawlers_successful():
|
|
return r_db.hget('crawler:tor:test', 'success') == 'True'
|
|
|
|
|
|
def get_test_ail_crawlers_message():
|
|
return r_db.hget('crawler:tor:test', 'message')
|
|
|
|
|
|
def save_test_ail_crawlers_result(test_success, message):
|
|
r_db.hset('crawler:tor:test', 'success', str(test_success))
|
|
r_db.hset('crawler:tor:test', 'message', message)
|
|
|
|
def test_ail_crawlers():
|
|
# # TODO: test web domain
|
|
if not ping_lacus():
|
|
lacus_url = get_lacus_url()
|
|
error_message = f'Error: Can\'t connect to AIL Lacus, {lacus_url}'
|
|
print(error_message)
|
|
save_test_ail_crawlers_result(False, error_message)
|
|
return False
|
|
|
|
lacus = get_lacus()
|
|
commit_id = git_status.get_last_commit_id_from_local()
|
|
user_agent = f'commit_id-AIL LACUS CRAWLER'
|
|
domain = 'eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion'
|
|
url = 'http://eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion'
|
|
|
|
## LAUNCH CRAWLER, TEST MODE ##
|
|
# set_current_crawler_status(splash_url, 'CRAWLER TEST', started_time=True,
|
|
# crawled_domain='TEST DOMAIN', crawler_type='onion')
|
|
capture_uuid = lacus.enqueue(url=url, depth=0, user_agent=user_agent, proxy='force_tor',
|
|
force=True, general_timeout_in_sec=90)
|
|
status = lacus.get_capture_status(capture_uuid)
|
|
launch_time = int(time.time()) # capture timeout
|
|
while int(time.time()) - launch_time < 60 and status != CaptureStatus.DONE:
|
|
# DEBUG
|
|
print(int(time.time()) - launch_time)
|
|
print(status)
|
|
time.sleep(1)
|
|
status = lacus.get_capture_status(capture_uuid)
|
|
|
|
# TODO CRAWLER STATUS OR QUEUED CAPTURE LIST
|
|
entries = lacus.get_capture(capture_uuid)
|
|
if 'error' in entries:
|
|
save_test_ail_crawlers_result(False, entries['error'])
|
|
return False
|
|
elif 'html' in entries and entries['html']:
|
|
mess = 'It works!'
|
|
if mess in entries['html']:
|
|
save_test_ail_crawlers_result(True, mess)
|
|
return True
|
|
else:
|
|
return False
|
|
return False
|
|
|
|
#### ---- ####
|
|
|
|
#### CRAWLER PROXY ####
|
|
|
|
#### ---- ####
|
|
|
|
|
|
# TODO MOVE ME
|
|
load_blacklist()
|
|
|
|
if __name__ == '__main__':
|
|
# res = get_splash_manager_version()
|
|
# res = test_ail_crawlers()
|
|
# res = is_test_ail_crawlers_successful()
|
|
# print(res)
|
|
# print(get_test_ail_crawlers_message())
|
|
# print(get_all_queues_stats())
|
|
|
|
# res = get_auto_crawler_all_domain()
|
|
# res = get_all_cookiejar()
|
|
res = unpack_url('http://test.com/')
|
|
print(res)
|