AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] fix crawler queue stats
Some checks are pending
CI / ail_test (3.10) (push) Waiting to run
Details
CI / ail_test (3.7) (push) Waiting to run
Details
CI / ail_test (3.8) (push) Waiting to run
Details
CI / ail_test (3.9) (push) Waiting to run
Details

Browse source
This commit is contained in:
terrtia 2024-09-17 15:36:15 +02:00
parent cc7e67d5ed
commit a20b6054e8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
2 changed files with 12 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/crawlers/Crawler.py
View file

@ -61,6 +61,8 @@ class Crawler(AbstractModule):
crawlers.load_blacklist()
# update captures cache
crawlers.reload_crawler_captures()
# update crawler queue stats
crawlers.reload_crawlers_stats()
self.crawler_scheduler = crawlers.CrawlerScheduler()

10
bin/lib/crawlers.py
View file

@ -1018,6 +1018,16 @@ def get_crawlers_stats(domain_type=None):
stats[domain_type] = {'queue': queue, 'up': up, 'down': down, 'crawled': crawled}
return stats
def reload_crawlers_stats():
for domain_type in get_crawler_all_types():
to_remove = []
for task_uuid in r_crawler.smembers(f'crawler:queue:type:{domain_type}'):
task = CrawlerTask(task_uuid)
if not task.exists():
to_remove.append(task_uuid)
for task_uuid in to_remove:
r_crawler.srem(f'crawler:queue:type:{domain_type}', task_uuid)
#### Blocklist ####
def get_blacklist():