AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-27 00:07:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge

Browse source
This commit is contained in:
Terrtia 2021-07-14 14:08:15 +02:00
commit b29767a020
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 11 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bin/lib/crawlers.py
View file

@ -583,7 +583,7 @@ def api_set_nb_crawlers_to_launch(dict_splash_name):
# TODO: check if is dict # TODO: check if is dict
dict_crawlers_to_launch = {} dict_crawlers_to_launch = {}
all_splash = get_all_splash() all_splash = get_all_splash()
crawlers_to_launch = list(all_splash & set(dict_splash_name.keys())) crawlers_to_launch = list(set(all_splash) & set(dict_splash_name.keys()))
for splash_name in crawlers_to_launch: for splash_name in crawlers_to_launch:
try: try:
nb_to_launch = int(dict_splash_name.get(splash_name, 0)) nb_to_launch = int(dict_splash_name.get(splash_name, 0))
@ -1147,26 +1147,24 @@ def api_save_splash_manager_url_api(data):
## SPLASH ## ## SPLASH ##
def get_all_splash(r_list=False): def get_all_splash(r_list=False):
res = r_serv_onion.smembers('all_splash') res = r_serv_onion.smembers('all_splash')
if res: if not res:
res = set()
if r_list: if r_list:
return list(res) return list(res)
else: else:
return res return res
else:
return []
def get_splash_proxy(splash_name): def get_splash_proxy(splash_name):
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy') return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy')
def get_splash_all_url(splash_name, r_list=False): def get_splash_all_url(splash_name, r_list=False):
res = r_serv_onion.smembers('splash:url:{}'.format(splash_name)) res = r_serv_onion.smembers('splash:url:{}'.format(splash_name))
if res: if not res:
res = set()
if r_list: if r_list:
return list(res) return list(res)
else: else:
return res return res
else:
return []
def get_splash_name_by_url(splash_url): def get_splash_name_by_url(splash_url):
return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url)) return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url))