chg: [crawlers manager] show setings

This commit is contained in:
Terrtia 2020-08-18 19:10:38 +02:00
parent 3ea14b29b8
commit 65f6ee4911
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 285 additions and 5 deletions

View file

@ -667,6 +667,13 @@ def get_splash_manager_url(reload=False): # TODO: add in db config
def get_splash_api_key(reload=False): # TODO: add in db config def get_splash_api_key(reload=False): # TODO: add in db config
return splash_api_key return splash_api_key
def get_hidden_splash_api_key(): # TODO: add in db config
key = get_splash_api_key()
if len(key)==41:
return f'{key[:4]}*********************************{key[-4:]}'
else:
return None
def get_splash_url_from_manager_url(splash_manager_url, splash_port): def get_splash_url_from_manager_url(splash_manager_url, splash_port):
url = urlparse(splash_manager_url) url = urlparse(splash_manager_url)
host = url.netloc.split(':', 1)[0] host = url.netloc.split(':', 1)[0]
@ -780,7 +787,23 @@ def get_splash_name_by_url(splash_url):
def get_splash_crawler_type(splash_name): def get_splash_crawler_type(splash_name):
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'crawler_type') return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'crawler_type')
def get_all_splash_by_proxy(proxy_name): def get_splash_crawler_description(splash_name):
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'description')
def get_splash_crawler_metadata(splash_name):
dict_splash = {}
dict_splash['proxy'] = get_splash_proxy(splash_name)
dict_splash['type'] = get_splash_crawler_type(splash_name)
dict_splash['description'] = get_splash_crawler_description(splash_name)
return dict_splash
def get_all_splash_crawler_metadata():
dict_splash = {}
for splash_name in get_all_splash():
dict_splash[splash_name] = get_splash_crawler_metadata(splash_name)
return dict_splash
def get_all_splash_by_proxy(proxy_name, r_list=False):
res = r_serv_onion.smembers('proxy:splash:{}'.format(proxy_name)) res = r_serv_onion.smembers('proxy:splash:{}'.format(proxy_name))
if res: if res:
if r_list: if r_list:
@ -816,6 +839,36 @@ def delete_all_proxies():
for proxy_name in get_all_proxies(): for proxy_name in get_all_proxies():
delete_proxy(proxy_name) delete_proxy(proxy_name)
def get_proxy_host(proxy_name):
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'host')
def get_proxy_port(proxy_name):
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'port')
def get_proxy_type(proxy_name):
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'type')
def get_proxy_crawler_type(proxy_name):
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'crawler_type')
def get_proxy_description(proxy_name):
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'description')
def get_proxy_metadata(proxy_name):
meta_dict = {}
meta_dict['host'] = get_proxy_host(proxy_name)
meta_dict['port'] = get_proxy_port(proxy_name)
meta_dict['type'] = get_proxy_type(proxy_name)
meta_dict['crawler_type'] = get_proxy_crawler_type(proxy_name)
meta_dict['description'] = get_proxy_description(proxy_name)
return meta_dict
def get_all_proxies_metadata():
all_proxy_dict = {}
for proxy_name in get_all_proxies():
all_proxy_dict[proxy_name] = get_proxy_metadata(proxy_name)
return all_proxy_dict
def set_proxy_used_in_discovery(proxy_name, value): def set_proxy_used_in_discovery(proxy_name, value):
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value) r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value)
@ -851,6 +904,7 @@ def load_all_splash_containers():
r_serv_onion.set('splash:map:url:name:{}'.format(splash_url), splash_name) r_serv_onion.set('splash:map:url:name:{}'.format(splash_url), splash_name)
def load_all_proxy(): def load_all_proxy():
delete_all_proxies()
all_proxies = get_all_splash_manager_proxies() all_proxies = get_all_splash_manager_proxies()
for proxy_name in all_proxies: for proxy_name in all_proxies:
proxy_dict = all_proxies[proxy_name] proxy_dict = all_proxies[proxy_name]
@ -861,6 +915,7 @@ def load_all_proxy():
description = all_proxies[proxy_name].get('description', None) description = all_proxies[proxy_name].get('description', None)
if description: if description:
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'description', description) r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'description', description)
r_serv_onion.sadd('all_proxy', proxy_name)
def reload_splash_and_proxies_list(): def reload_splash_and_proxies_list():
if ping_splash_manager(): if ping_splash_manager():

View file

@ -428,11 +428,20 @@ def crawler_cookiejar_cookie_json_add_post():
return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid)) return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid))
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['GET']) @crawler_splash.route('/crawler/settings', methods=['GET'])
@login_required @login_required
@login_analyst @login_analyst
def crawler_splash_setings(): def crawler_splash_setings():
all_proxies = crawlers.get_all_proxies_metadata()
return render_template("settings_splash_crawler.html", cookiejar_uuid=True, cookie_uuid=False) all_splash = crawlers.get_all_splash_crawler_metadata()
splash_manager_url = crawlers.get_splash_manager_url()
api_key = crawlers.get_hidden_splash_api_key()
is_manager_connected = crawlers.ping_splash_manager()
return render_template("settings_splash_crawler.html",
is_manager_connected=is_manager_connected,
splash_manager_url=splash_manager_url, api_key=api_key,
all_splash=all_splash, all_proxies=all_proxies)
## - - ## ## - - ##

View file

@ -0,0 +1,216 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{%if not is_manager_connected%}
{% include 'crawler/crawler_disabled.html' %}
{%endif%}
<div class="row">
<div class="col-xl-6">
</div>
<div class="col-xl-6">
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-header bg-dark text-white">
<span class="badge badge-pill badge-light flex-row-reverse float-right">
{% if is_manager_connected %}
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
Connected
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle fa-2x"></i>
Error
</div>
{% endif %}
</span>
<h4>Splash Crawler Manager</h4>
</div>
<div class="card-body">
<div class="row mb-3 justify-content-center">
<div class="col-xl-6">
<div class="card text-center border-secondary">
<div class="card-body px-1 py-0">
<table class="table table-sm">
<tbody>
<tr>
<td>Splash Manager URL</td>
<td>{{splash_manager_url}}</td>
</tr>
<tr>
<td>API Key</td>
<td>
{{api_key}}
<!-- <a class="ml-3" href="/settings/new_token"><i class="fa fa-random"></i></a> -->
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="card border-secondary mb-4">
<div class="card-body text-dark">
<h5 class="card-title">All Splash Crawlers:</h5>
<table class="table table-striped">
<thead class="bg-info text-white">
<th>
Splash name
</th>
<th>
Proxy
</th>
<th>
Crawler type
</th>
<th>
Description
</th>
</thead>
<tbody>
{% for splash_name in all_splash %}
<tr>
<td>
{{splash_name}}
</td>
<td>
{{all_splash[splash_name]['proxy']}}
</td>
<td>
{%if all_splash[splash_name]['type']=='tor'%}
<i class="fas fa-user-secret"></i>
{%else%}
<i class="fab fa-html5">
{%endif%}
{{all_splash[splash_name]['type']}}
</td>
<td>
{{all_splash[splash_name]['description']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="card border-secondary">
<div class="card-body text-dark">
<h5 class="card-title">All Proxies:</h5>
<table class="table table-striped">
<thead class="bg-info text-white">
<th>
Proxy name
</th>
<th>
Host
</th>
<th>
Port
</th>
<th>
Type
</th>
<th>
Crawler Type
</th>
<th>
Description
</th>
</thead>
<tbody>
{% for proxy_name in all_proxies %}
<tr>
<td>
{{proxy_name}}
</td>
<td>
{{all_proxies[proxy_name]['host']}}
</td>
<td>
{{all_proxies[proxy_name]['port']}}
</td>
<td>
{{all_proxies[proxy_name]['type']}}
</td>
<td>
{%if all_proxies[proxy_name]['crawler_type']=='tor'%}
<i class="fas fa-user-secret"></i>
{%else%}
<i class="fab fa-html5">
{%endif%}
{{all_proxies[proxy_name]['crawler_type']}}
</td>
<td>
{{all_proxies[proxy_name]['description']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-header bg-dark text-white">
<h4>Crawlers Settings</h4>
</div>
<div class="card-body">
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var to_refresh = false
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_settings").addClass("active");
});
</script>

View file

@ -44,7 +44,7 @@
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hiddenServices.auto_crawler')}}" id="nav_settings"> <a class="nav-link" href="{{url_for('crawler_splash.crawler_splash_setings')}}" id="nav_settings">
<i class="fas fa-cog"></i> <i class="fas fa-cog"></i>
Settings Settings
</a> </a>