AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 16:36:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] fix reload_crawlers_stats queues stats

Browse source
This commit is contained in:
terrtia 2024-12-16 10:59:17 +01:00
parent 5b5ee502d0
commit 3faea7997c
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
bin/lib/crawlers.py
View file

@ -1101,8 +1101,14 @@ def reload_crawlers_stats():
tasks = r_crawler.smembers(f'crawler:queue:type:{domain_type}')
for task_uuid in tasks:
task = CrawlerTask(task_uuid)
if not task.is_in_queue() and task.get_status() is None and not task.get_capture():
task.delete()
if not task.is_in_queue() and task.get_status() is None:
capture = task.get_capture()
if capture:
c = CrawlerCapture(capture)
if not c.exists():
task.delete()
else:
task.delete()
#### Blocklist ####