AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-26 15:57:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [Crawler splash ResponseNeverReceived] add retry

Browse source
This commit is contained in:
Terrtia 2020-04-06 10:52:44 +02:00
parent 00573c9401
commit 672bb02bbf
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 20 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
bin/torcrawler/TorSplashCrawler.py
View file

@ -172,7 +172,26 @@ class TorSplashCrawler():
# LUA ERROR # # TODO: print/display errors
elif 'error' in response.data:
if(response.data['error'] == 'network99'):
print('Connection to proxy refused')
## splash restart ##
error_retry = request.meta.get('error_retry', 0)
if error_retry < 3:
error_retry += 1
url= request.meta['current_url']
father = request.meta['father']
self.logger.error('Splash, ResponseNeverReceived for %s, retry in 10s ...', url)
time.sleep(10)
yield SplashRequest(
url,
self.parse,
errback=self.errback_catcher,
endpoint='execute',
cache_args=['lua_source'],
meta={'father': father, 'current_url': url, 'error_retry' = error_retry},
args=self.build_request_arg(response.cookiejar)
)
else:
print('Connection to proxy refused')
else:
print(response.data['error'])