From 709d5487b86af3171e0278c139886767492dd255 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 13 Feb 2019 16:53:38 +0100 Subject: [PATCH] fix: [Onion] filter too many subdomain --- bin/Onion.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/Onion.py b/bin/Onion.py index d15875e4..d55d3506 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -224,6 +224,10 @@ if __name__ == "__main__": else: continue + # too many subdomain + if len(domain.split('.')) > 5: + continue + if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain): if not r_onion.sismember('onion_domain_crawler_queue', domain): print('send to onion crawler')