AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-03-18 20:39:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #350 from kovacsbalu/fix-crawler-rotation

Browse source 
fix: [crawler] rotation
This commit is contained in:
Thirion Aurélien 2019-05-16 13:31:09 +02:00 • committed by GitHub
commit e09d49a658
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/Crawler.py
View file

@ -12,6 +12,7 @@ import time
import subprocess
import requests
from collections import deque
from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN'])
@ -303,7 +304,7 @@ if __name__ == '__main__':
#mode = sys.argv[1]
splash_port = sys.argv[1]
rotation_mode = ['onion', 'regular']
rotation_mode = deque(['onion', 'regular'])
default_proto_map = {'http': 80, 'https': 443}
######################################################## add ftp ???
@ -361,6 +362,7 @@ if __name__ == '__main__':
update_auto_crawler()
rotation_mode.rotate()
to_crawl = get_elem_to_crawl(rotation_mode)
if to_crawl:
url_data = unpack_url(to_crawl['url'])