mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-13 01:58:22 +00:00
fix: [Crawler] fix crawler cache info
This commit is contained in:
parent
bb301a870c
commit
2dc0eca4a9
1 changed files with 8 additions and 2 deletions
|
@ -42,7 +42,7 @@ def crawl_onion(url, domain, date, date_month, message):
|
||||||
# TODO: relaunch docker or send error message
|
# TODO: relaunch docker or send error message
|
||||||
nb_retry += 1
|
nb_retry += 1
|
||||||
|
|
||||||
if nb_retry == 30:
|
if nb_retry == 6:
|
||||||
on_error_send_message_back_in_queue(type_hidden_service, domain, message)
|
on_error_send_message_back_in_queue(type_hidden_service, domain, message)
|
||||||
publisher.error('{} SPASH DOWN'.format(splash_url))
|
publisher.error('{} SPASH DOWN'.format(splash_url))
|
||||||
print('--------------------------------------')
|
print('--------------------------------------')
|
||||||
|
@ -187,6 +187,8 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
domain_url = 'http://{}'.format(domain)
|
domain_url = 'http://{}'.format(domain)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print()
|
||||||
print('\033[92m------------------START CRAWLER------------------\033[0m')
|
print('\033[92m------------------START CRAWLER------------------\033[0m')
|
||||||
print('crawler type: {}'.format(type_hidden_service))
|
print('crawler type: {}'.format(type_hidden_service))
|
||||||
print('\033[92m-------------------------------------------------\033[0m')
|
print('\033[92m-------------------------------------------------\033[0m')
|
||||||
|
@ -262,7 +264,11 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
#update crawler status
|
#update crawler status
|
||||||
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting')
|
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting')
|
||||||
r_cache.hrem('metadata_crawler:{}'.format(splash_port), 'crawling_domain')
|
r_cache.hdel('metadata_crawler:{}'.format(splash_port), 'crawling_domain')
|
||||||
|
else:
|
||||||
|
print(' Blacklisted Onion')
|
||||||
|
print()
|
||||||
|
print()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
|
|
Loading…
Reference in a new issue