AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 16:36:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] increase timeout QUEUED captures
Some checks are pending
CI / ail_test (3.10) (push) Waiting to run
Details
CI / ail_test (3.7) (push) Waiting to run
Details
CI / ail_test (3.8) (push) Waiting to run
Details
CI / ail_test (3.9) (push) Waiting to run
Details

Browse source
This commit is contained in:
terrtia 2025-01-13 10:11:01 +01:00
parent aa2113bc36
commit afd0d6b7d8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/crawlers/Crawler.py
View file

@ -180,11 +180,11 @@ class Crawler(AbstractModule):
capture.update(status) capture.update(status)
elif status == crawlers.CaptureStatus.QUEUED: elif status == crawlers.CaptureStatus.QUEUED:
capture_start = capture.get_start_time(r_str=False) capture_start = capture.get_start_time(r_str=False)
if int(time.time()) - capture_start > 3600: # TODO ADD in new crawler config if int(time.time()) - capture_start > 36000: # TODO ADD in new crawler config
task = capture.get_task() task = capture.get_task()
task.reset() task.reset()
capture.delete() capture.delete()
self.logger.warning(f'capture QUEUED Timeout, {task.uuid} Send back in queue') self.logger.warning(f'capture QUEUED Timeout, {task.uuid} Send back in queue, start_time={capture_start}')
else: else:
capture.update(status) capture.update(status)
print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))