AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-09-20 07:58:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix: [Crawler] fix index + redis history key

Browse source
This commit is contained in:
Terrtia 2019-03-22 17:14:27 +01:00
parent f64c385343
commit 5165a5de2f
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
bin/packages/HiddenServices.py
View file

@ -114,14 +114,14 @@ class HiddenServices(object):
res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True) res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
if res: if res:
res = res[0] res = res[0]
return {'root_item':res[0], 'epoch':res[1]} return {'root_item':res[0], 'epoch':int(res[1])}
else: else:
return {} return {}
def get_last_crawled(self): def get_last_crawled(self):
res = self.r_serv_onion.zrevrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True) res = self.r_serv_onion.zrevrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
if res: if res:
return {'root_item':res[0], 'epoch':res[1]} return {'root_item':res[0][0], 'epoch':res[0][1]}
else: else:
return {} return {}
@ -129,7 +129,7 @@ class HiddenServices(object):
def get_domain_crawled_core_item(self, epoch=None): def get_domain_crawled_core_item(self, epoch=None):
core_item = {} core_item = {}
if epoch: if epoch:
list_root = self.r_serv_onion.zrevrangebyscore('crawler_history_{}:{}'.format(self.type, self.domain, self.port), int(epoch), int(epoch)) list_root = self.r_serv_onion.zrevrangebyscore('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), int(epoch), int(epoch))
if list_root: if list_root:
core_item['root_item'] = list_root[0] core_item['root_item'] = list_root[0]
core_item['epoch'] = epoch core_item['epoch'] = epoch