AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [crawler] filter lookup tags
Some checks are pending
CI / ail_test (3.10) (push) Waiting to run
Details
CI / ail_test (3.7) (push) Waiting to run
Details
CI / ail_test (3.8) (push) Waiting to run
Details
CI / ail_test (3.9) (push) Waiting to run
Details

Browse source
This commit is contained in:
terrtia 2024-10-07 14:53:13 +02:00
parent 83e11082b5
commit d9fc014a1d
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 5 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
bin/lib/crawlers.py
View file

@ -87,6 +87,11 @@ def api_get_onion_lookup(domain): # TODO check if object process done ???
meta['first_seen'] = meta['first_seen'].replace('/', '-') meta['first_seen'] = meta['first_seen'].replace('/', '-')
meta['last_seen'] = meta['last_check'].replace('/', '-') meta['last_seen'] = meta['last_check'].replace('/', '-')
meta['languages'] = list(meta['languages']) meta['languages'] = list(meta['languages'])
tags = []
for tag in meta['tags']:
if tag.startswith('dark-web:') or tag.startswith('infoleak:'):
tags.append(tag)
meta['tags'] = tags
del meta['domain'] del meta['domain']
del meta['last_check'] del meta['last_check']
del meta['type'] del meta['type']