mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [crawlers manager] show setings
This commit is contained in:
Terrtia
parent
3ea14b29b8
commit
65f6ee4911
4 changed files with 285 additions and 5 deletions
|
|
@ -667,6 +667,13 @@ def get_splash_manager_url(reload=False): # TODO: add in db config
|
|||
def get_splash_api_key(reload=False): # TODO: add in db config
|
||||
return splash_api_key
|
||||
|
||||
def get_hidden_splash_api_key(): # TODO: add in db config
|
||||
key = get_splash_api_key()
|
||||
if len(key)==41:
|
||||
return f'{key[:4]}*********************************{key[-4:]}'
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_splash_url_from_manager_url(splash_manager_url, splash_port):
|
||||
url = urlparse(splash_manager_url)
|
||||
host = url.netloc.split(':', 1)[0]
|
||||
|
|
@ -780,7 +787,23 @@ def get_splash_name_by_url(splash_url):
|
|||
def get_splash_crawler_type(splash_name):
|
||||
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'crawler_type')
|
||||
|
||||
def get_all_splash_by_proxy(proxy_name):
|
||||
def get_splash_crawler_description(splash_name):
|
||||
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'description')
|
||||
|
||||
def get_splash_crawler_metadata(splash_name):
|
||||
dict_splash = {}
|
||||
dict_splash['proxy'] = get_splash_proxy(splash_name)
|
||||
dict_splash['type'] = get_splash_crawler_type(splash_name)
|
||||
dict_splash['description'] = get_splash_crawler_description(splash_name)
|
||||
return dict_splash
|
||||
|
||||
def get_all_splash_crawler_metadata():
|
||||
dict_splash = {}
|
||||
for splash_name in get_all_splash():
|
||||
dict_splash[splash_name] = get_splash_crawler_metadata(splash_name)
|
||||
return dict_splash
|
||||
|
||||
def get_all_splash_by_proxy(proxy_name, r_list=False):
|
||||
res = r_serv_onion.smembers('proxy:splash:{}'.format(proxy_name))
|
||||
if res:
|
||||
if r_list:
|
||||
|
|
@ -816,6 +839,36 @@ def delete_all_proxies():
|
|||
for proxy_name in get_all_proxies():
|
||||
delete_proxy(proxy_name)
|
||||
|
||||
def get_proxy_host(proxy_name):
|
||||
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'host')
|
||||
|
||||
def get_proxy_port(proxy_name):
|
||||
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'port')
|
||||
|
||||
def get_proxy_type(proxy_name):
|
||||
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'type')
|
||||
|
||||
def get_proxy_crawler_type(proxy_name):
|
||||
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'crawler_type')
|
||||
|
||||
def get_proxy_description(proxy_name):
|
||||
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'description')
|
||||
|
||||
def get_proxy_metadata(proxy_name):
|
||||
meta_dict = {}
|
||||
meta_dict['host'] = get_proxy_host(proxy_name)
|
||||
meta_dict['port'] = get_proxy_port(proxy_name)
|
||||
meta_dict['type'] = get_proxy_type(proxy_name)
|
||||
meta_dict['crawler_type'] = get_proxy_crawler_type(proxy_name)
|
||||
meta_dict['description'] = get_proxy_description(proxy_name)
|
||||
return meta_dict
|
||||
|
||||
def get_all_proxies_metadata():
|
||||
all_proxy_dict = {}
|
||||
for proxy_name in get_all_proxies():
|
||||
all_proxy_dict[proxy_name] = get_proxy_metadata(proxy_name)
|
||||
return all_proxy_dict
|
||||
|
||||
def set_proxy_used_in_discovery(proxy_name, value):
|
||||
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value)
|
||||
|
||||
|
|
@ -851,6 +904,7 @@ def load_all_splash_containers():
|
|||
r_serv_onion.set('splash:map:url:name:{}'.format(splash_url), splash_name)
|
||||
|
||||
def load_all_proxy():
|
||||
delete_all_proxies()
|
||||
all_proxies = get_all_splash_manager_proxies()
|
||||
for proxy_name in all_proxies:
|
||||
proxy_dict = all_proxies[proxy_name]
|
||||
|
|
@ -861,6 +915,7 @@ def load_all_proxy():
|
|||
description = all_proxies[proxy_name].get('description', None)
|
||||
if description:
|
||||
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'description', description)
|
||||
r_serv_onion.sadd('all_proxy', proxy_name)
|
||||
|
||||
def reload_splash_and_proxies_list():
|
||||
if ping_splash_manager():
|
||||
|
|
|
|||
|
|
@ -428,11 +428,20 @@ def crawler_cookiejar_cookie_json_add_post():
|
|||
|
||||
return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid))
|
||||
|
||||
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['GET'])
|
||||
@crawler_splash.route('/crawler/settings', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def crawler_splash_setings():
|
||||
|
||||
return render_template("settings_splash_crawler.html", cookiejar_uuid=True, cookie_uuid=False)
|
||||
all_proxies = crawlers.get_all_proxies_metadata()
|
||||
all_splash = crawlers.get_all_splash_crawler_metadata()
|
||||
|
||||
splash_manager_url = crawlers.get_splash_manager_url()
|
||||
api_key = crawlers.get_hidden_splash_api_key()
|
||||
is_manager_connected = crawlers.ping_splash_manager()
|
||||
|
||||
return render_template("settings_splash_crawler.html",
|
||||
is_manager_connected=is_manager_connected,
|
||||
splash_manager_url=splash_manager_url, api_key=api_key,
|
||||
all_splash=all_splash, all_proxies=all_proxies)
|
||||
|
||||
## - - ##
|
||||
|
|
|
|||
|
|
@ -0,0 +1,216 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'crawler/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{%if not is_manager_connected%}
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
{%endif%}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="card mb-3 mt-1">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<span class="badge badge-pill badge-light flex-row-reverse float-right">
|
||||
{% if is_manager_connected %}
|
||||
<div style="color:Green;">
|
||||
<i class="fas fa-check-circle fa-2x"></i>
|
||||
Connected
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="color:Red;">
|
||||
<i class="fas fa-times-circle fa-2x"></i>
|
||||
Error
|
||||
</div>
|
||||
{% endif %}
|
||||
</span>
|
||||
<h4>Splash Crawler Manager</h4>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<div class="row mb-3 justify-content-center">
|
||||
<div class="col-xl-6">
|
||||
<div class="card text-center border-secondary">
|
||||
<div class="card-body px-1 py-0">
|
||||
<table class="table table-sm">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>Splash Manager URL</td>
|
||||
<td>{{splash_manager_url}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>API Key</td>
|
||||
<td>
|
||||
{{api_key}}
|
||||
<!-- <a class="ml-3" href="/settings/new_token"><i class="fa fa-random"></i></a> -->
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card border-secondary mb-4">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">All Splash Crawlers:</h5>
|
||||
<table class="table table-striped">
|
||||
<thead class="bg-info text-white">
|
||||
<th>
|
||||
Splash name
|
||||
</th>
|
||||
<th>
|
||||
Proxy
|
||||
</th>
|
||||
<th>
|
||||
Crawler type
|
||||
</th>
|
||||
<th>
|
||||
Description
|
||||
</th>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for splash_name in all_splash %}
|
||||
<tr>
|
||||
<td>
|
||||
{{splash_name}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_splash[splash_name]['proxy']}}
|
||||
</td>
|
||||
<td>
|
||||
{%if all_splash[splash_name]['type']=='tor'%}
|
||||
<i class="fas fa-user-secret"></i>
|
||||
{%else%}
|
||||
<i class="fab fa-html5">
|
||||
{%endif%}
|
||||
{{all_splash[splash_name]['type']}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_splash[splash_name]['description']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card border-secondary">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">All Proxies:</h5>
|
||||
<table class="table table-striped">
|
||||
<thead class="bg-info text-white">
|
||||
<th>
|
||||
Proxy name
|
||||
</th>
|
||||
<th>
|
||||
Host
|
||||
</th>
|
||||
<th>
|
||||
Port
|
||||
</th>
|
||||
<th>
|
||||
Type
|
||||
</th>
|
||||
<th>
|
||||
Crawler Type
|
||||
</th>
|
||||
<th>
|
||||
Description
|
||||
</th>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for proxy_name in all_proxies %}
|
||||
<tr>
|
||||
<td>
|
||||
{{proxy_name}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_proxies[proxy_name]['host']}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_proxies[proxy_name]['port']}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_proxies[proxy_name]['type']}}
|
||||
</td>
|
||||
<td>
|
||||
{%if all_proxies[proxy_name]['crawler_type']=='tor'%}
|
||||
<i class="fas fa-user-secret"></i>
|
||||
{%else%}
|
||||
<i class="fab fa-html5">
|
||||
{%endif%}
|
||||
{{all_proxies[proxy_name]['crawler_type']}}
|
||||
</td>
|
||||
<td>
|
||||
{{all_proxies[proxy_name]['description']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card mb-3 mt-1">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h4>Crawlers Settings</h4>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
$("#page-Crawler").addClass("active");
|
||||
$("#nav_settings").addClass("active");
|
||||
});
|
||||
|
||||
</script>
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hiddenServices.auto_crawler')}}" id="nav_settings">
|
||||
<a class="nav-link" href="{{url_for('crawler_splash.crawler_splash_setings')}}" id="nav_settings">
|
||||
<i class="fas fa-cog"></i>
|
||||
Settings
|
||||
</a>
|
||||
|
|
|
|||
Loading…
Reference in a new issue