AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] fix new crawled item id

Browse source
This commit is contained in:
Terrtia 2021-07-14 15:48:17 +02:00
parent 25813a7287
commit 7a652b5195
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/lib/crawlers.py
View file

@ -775,6 +775,8 @@ def create_item_metadata(item_id, domain, url, port, item_father):
r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id) r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id)
def create_item_id(item_dir, domain): def create_item_id(item_dir, domain):
# remove /
domain = domain.replace('/', '_')
if len(domain) > 215: if len(domain) > 215:
UUID = domain[-215:]+str(uuid.uuid4()) UUID = domain[-215:]+str(uuid.uuid4())
else: else: