mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Fix crawler rotation
Before this, crawler processed prioritized onions and after all starts prioritized regular.
This commit is contained in:
parent
0389b9c23b
commit
6092f482e6
1 changed files with 3 additions and 1 deletions
|
@ -12,6 +12,7 @@ import time
|
|||
import subprocess
|
||||
import requests
|
||||
|
||||
from collections import deque
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
|
@ -303,7 +304,7 @@ if __name__ == '__main__':
|
|||
#mode = sys.argv[1]
|
||||
splash_port = sys.argv[1]
|
||||
|
||||
rotation_mode = ['onion', 'regular']
|
||||
rotation_mode = deque(["onion", "regular"])
|
||||
default_proto_map = {'http': 80, 'https': 443}
|
||||
######################################################## add ftp ???
|
||||
|
||||
|
@ -361,6 +362,7 @@ if __name__ == '__main__':
|
|||
|
||||
update_auto_crawler()
|
||||
|
||||
rotation_mode.rotate()
|
||||
to_crawl = get_elem_to_crawl(rotation_mode)
|
||||
if to_crawl:
|
||||
url_data = unpack_url(to_crawl['url'])
|
||||
|
|
Loading…
Reference in a new issue