diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 99ed03a4..e0890443 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -147,6 +147,7 @@ class Crawler(AbstractModule): if capture: try: status = self.lacus.get_capture_status(capture.uuid) + print(status) if status == crawlers.CaptureStatus.DONE: return capture elif status == crawlers.CaptureStatus.UNKNOWN: @@ -164,7 +165,10 @@ class Crawler(AbstractModule): self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue') else: capture.update(status) - elif status == crawlers.CaptureStatus.QUEUED or status == crawlers.CaptureStatus.ONGOING: + elif status == crawlers.CaptureStatus.QUEUED: + capture.update(status, delta=30) + print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time() + 30)) + elif status == crawlers.CaptureStatus.ONGOING: capture.update(status) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) # Invalid State diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 4534a702..eac5c8b0 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1531,13 +1531,13 @@ class CrawlerCapture: r_crawler.zadd('crawler:captures', {self.uuid: launch_time}) r_cache.zadd('crawler:captures', {self.uuid: launch_time}) - def update(self, status): + def update(self, status, delta=0): # Error or Reload if not status: r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN.value) r_cache.zadd('crawler:captures', {self.uuid: 0}) else: - last_check = int(time.time()) + last_check = int(time.time() + delta) r_cache.hset(f'crawler:capture:{self.uuid}', 'status', status) r_cache.zadd('crawler:captures', {self.uuid: last_check})