AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-19 00:36:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [Onion] filter too many subdomain

Browse source
This commit is contained in:
Terrtia 2019-02-13 16:53:38 +01:00
parent b87707e8bc
commit 709d5487b8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/Onion.py
View file

@ -224,6 +224,10 @@ if __name__ == "__main__":
else: else:
continue continue
# too many subdomain
if len(domain.split('.')) > 5:
continue
if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain): if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain):
if not r_onion.sismember('onion_domain_crawler_queue', domain): if not r_onion.sismember('onion_domain_crawler_queue', domain):
print('send to onion crawler') print('send to onion crawler')