From df161cfd64b92fb4635015a8f42ba73cf2d9b02b Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 9 Jan 2025 11:18:29 +0100 Subject: [PATCH] fix: [crawler] timeout QUEUED captures --- bin/crawlers/Crawler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 309b649a..e7e780ed 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -179,7 +179,14 @@ class Crawler(AbstractModule): else: capture.update(status) elif status == crawlers.CaptureStatus.QUEUED: - capture.update(status) + capture_start = capture.get_start_time(r_str=False) + if int(time.time()) - capture_start > 600: # TODO ADD in new crawler config + task = capture.get_task() + task.reset() + capture.delete() + self.logger.warning(f'capture QUEUED Timeout, {task.uuid} Send back in queue') + else: + capture.update(status) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) elif status == crawlers.CaptureStatus.ONGOING: capture.update(status)