fix: [Onion] filter too many subdomain

This commit is contained in:
Terrtia 2019-02-13 16:53:38 +01:00
parent b87707e8bc
commit 709d5487b8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0

View file

@ -224,6 +224,10 @@ if __name__ == "__main__":
else:
continue
# too many subdomain
if len(domain.split('.')) > 5:
continue
if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain):
if not r_onion.sismember('onion_domain_crawler_queue', domain):
print('send to onion crawler')