AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-14 02:28:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] fix auto crawler creation

Browse source
This commit is contained in:
Terrtia 2020-07-31 17:22:41 +02:00
parent 0b44ae86cd
commit a88b57498d
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/lib/crawlers.py
View file

@ -428,7 +428,7 @@ def save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=
if crawler_mode == 'manual': if crawler_mode == 'manual':
r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config)) r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config))
elif crawler_mode == 'auto': elif crawler_mode == 'auto':
r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_type, crawler_type, domain, url), json.dumps(crawler_config)) r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_mode, crawler_type, domain, url), json.dumps(crawler_config))
def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url): def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode)) r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode))