AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 16:36:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] debug crawler queued capture loop

Browse source
This commit is contained in:
terrtia 2025-01-07 15:23:06 +01:00
parent 80c7410cb1
commit e6e48c69f5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
2 changed files with 7 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
bin/crawlers/Crawler.py
View file

@ -147,6 +147,7 @@ class Crawler(AbstractModule):
if capture: if capture:
try: try:
status = self.lacus.get_capture_status(capture.uuid) status = self.lacus.get_capture_status(capture.uuid)
print(status)
if status == crawlers.CaptureStatus.DONE: if status == crawlers.CaptureStatus.DONE:
return capture return capture
elif status == crawlers.CaptureStatus.UNKNOWN: elif status == crawlers.CaptureStatus.UNKNOWN:
@ -164,7 +165,10 @@ class Crawler(AbstractModule):
self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue') self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue')
else: else:
capture.update(status) capture.update(status)
elif status == crawlers.CaptureStatus.QUEUED or status == crawlers.CaptureStatus.ONGOING: elif status == crawlers.CaptureStatus.QUEUED:
capture.update(status, delta=30)
print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time() + 30))
elif status == crawlers.CaptureStatus.ONGOING:
capture.update(status) capture.update(status)
print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
# Invalid State # Invalid State

4
bin/lib/crawlers.py
View file

@ -1531,13 +1531,13 @@ class CrawlerCapture:
r_crawler.zadd('crawler:captures', {self.uuid: launch_time}) r_crawler.zadd('crawler:captures', {self.uuid: launch_time})
r_cache.zadd('crawler:captures', {self.uuid: launch_time}) r_cache.zadd('crawler:captures', {self.uuid: launch_time})
def update(self, status): def update(self, status, delta=0):
# Error or Reload # Error or Reload
if not status: if not status:
r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN.value) r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN.value)
r_cache.zadd('crawler:captures', {self.uuid: 0}) r_cache.zadd('crawler:captures', {self.uuid: 0})
else: else:
last_check = int(time.time()) last_check = int(time.time() + delta)
r_cache.hset(f'crawler:capture:{self.uuid}', 'status', status) r_cache.hset(f'crawler:capture:{self.uuid}', 'status', status)
r_cache.zadd('crawler:captures', {self.uuid: last_check}) r_cache.zadd('crawler:captures', {self.uuid: last_check})