mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [crawler] show all crawlers type on dashboard
This commit is contained in:
Terrtia
parent
8901ffe989
commit
3ea14b29b8
9 changed files with 371 additions and 306 deletions
|
|
@ -349,7 +349,7 @@ if __name__ == '__main__':
|
|||
'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
|
||||
|
||||
# Track launched crawler
|
||||
r_cache.sadd('all_crawler', splash_url)
|
||||
r_cache.sadd('all_splash_crawlers', splash_url)
|
||||
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status', 'Waiting')
|
||||
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
|
||||
|
||||
|
|
@ -385,7 +385,7 @@ if __name__ == '__main__':
|
|||
'epoch': int(time.time())}
|
||||
|
||||
# Update crawler status type
|
||||
r_cache.sadd('{}_crawlers'.format(to_crawl['type_service']), splash_url)
|
||||
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
|
||||
|
||||
crawler_config = load_crawler_config(to_crawl['type_service'], url_data['domain'], to_crawl['paste'], to_crawl['url'], date)
|
||||
# check if default crawler
|
||||
|
|
@ -437,7 +437,7 @@ if __name__ == '__main__':
|
|||
r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'crawling_domain')
|
||||
|
||||
# Update crawler status type
|
||||
r_cache.srem('{}_crawlers'.format(to_crawl['type_service']), splash_url)
|
||||
r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
|
||||
|
||||
# add next auto Crawling in queue:
|
||||
if to_crawl['paste'] == 'auto':
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ def launch_crawlers():
|
|||
print('Please launch {} additional {} Dockers'.format( nb_crawlers - len(all_crawler_urls), splash_name))
|
||||
nb_crawlers = len(all_crawler_urls)
|
||||
|
||||
crawlers.reset_all_spash_crawler_status()
|
||||
|
||||
for i in range(0, int(nb_crawlers)):
|
||||
splash_url = all_crawler_urls[i]
|
||||
print(all_crawler_urls[i])
|
||||
|
|
@ -59,6 +61,8 @@ if __name__ == '__main__':
|
|||
|
||||
while True:
|
||||
|
||||
# # TODO: avoid multiple ping
|
||||
|
||||
# check if manager is connected
|
||||
if int(time.time()) - last_check > 60:
|
||||
is_manager_connected = crawlers.is_splash_manager_connected()
|
||||
|
|
|
|||
|
|
@ -46,9 +46,20 @@ config_loader = None
|
|||
|
||||
faup = Faup()
|
||||
|
||||
# # # # # # # #
|
||||
# #
|
||||
# COMMON #
|
||||
# #
|
||||
# # # # # # # #
|
||||
|
||||
def generate_uuid():
|
||||
return str(uuid.uuid4()).replace('-', '')
|
||||
|
||||
def get_current_date():
|
||||
return datetime.now().strftime("%Y%m%d")
|
||||
|
||||
##-- COMMON --#
|
||||
|
||||
################################################################################
|
||||
|
||||
# # TODO: handle prefix cookies
|
||||
|
|
@ -377,6 +388,55 @@ def api_create_cookie(user_id, cookiejar_uuid, cookie_dict):
|
|||
|
||||
#### ####
|
||||
|
||||
# # # # # # # #
|
||||
# #
|
||||
# CRAWLER #
|
||||
# #
|
||||
# # # # # # # #
|
||||
|
||||
#### CRAWLER GLOBAL ####
|
||||
|
||||
def get_all_spash_crawler_status():
|
||||
crawler_metadata = []
|
||||
all_crawlers = r_cache.smembers('all_splash_crawlers')
|
||||
for crawler in all_crawlers:
|
||||
crawler_metadata.append(get_splash_crawler_status(crawler))
|
||||
return crawler_metadata
|
||||
|
||||
def reset_all_spash_crawler_status():
|
||||
r_cache.delete('all_splash_crawlers')
|
||||
|
||||
def get_splash_crawler_status(spash_url):
|
||||
crawler_type = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'type')
|
||||
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'crawling_domain')
|
||||
started_time = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'started_time')
|
||||
status_info = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'status')
|
||||
crawler_info = '{} - {}'.format(spash_url, started_time)
|
||||
if status_info=='Waiting' or status_info=='Crawling':
|
||||
status=True
|
||||
else:
|
||||
status=False
|
||||
return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type}
|
||||
|
||||
def get_stats_last_crawled_domains(crawler_types, date):
|
||||
statDomains = {}
|
||||
for crawler_type in crawler_types:
|
||||
stat_type = {}
|
||||
stat_type['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(crawler_type, date))
|
||||
stat_type['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(crawler_type, date))
|
||||
stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down']
|
||||
stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type)
|
||||
statDomains[crawler_type] = stat_type
|
||||
return statDomains
|
||||
|
||||
# # TODO: handle custom proxy
|
||||
def get_splash_crawler_latest_stats():
|
||||
now = datetime.now()
|
||||
date = now.strftime("%Y%m%d")
|
||||
return get_stats_last_crawled_domains(['onion', 'regular'], date)
|
||||
|
||||
##-- CRAWLER GLOBAL --##
|
||||
|
||||
#### CRAWLER TASK ####
|
||||
def create_crawler_task(url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, cookiejar_uuid=None, user_agent=None):
|
||||
|
||||
|
|
@ -587,10 +647,20 @@ def get_elem_to_crawl_by_queue_type(l_queue_type):
|
|||
return {'url': url, 'paste': item_id, 'type_service': queue_type, 'original_message': message}
|
||||
return None
|
||||
|
||||
def get_nb_elem_to_crawl_by_type(queue_type):
|
||||
nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type))
|
||||
nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type))
|
||||
nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type))
|
||||
return nb
|
||||
|
||||
#### ---- ####
|
||||
|
||||
# # # # # # # # # # # #
|
||||
# #
|
||||
# SPLASH MANAGER #
|
||||
# #
|
||||
# # # # # # # # # # # #
|
||||
|
||||
#### SPLASH MANAGER ####
|
||||
def get_splash_manager_url(reload=False): # TODO: add in db config
|
||||
return splash_manager_url
|
||||
|
||||
|
|
@ -636,6 +706,8 @@ def ping_splash_manager():
|
|||
return True
|
||||
else:
|
||||
print(req.json())
|
||||
update_splash_manager_connection_status(False)
|
||||
return False
|
||||
except requests.exceptions.ConnectionError:
|
||||
pass
|
||||
# splash manager unreachable
|
||||
|
|
|
|||
|
|
@ -48,6 +48,31 @@ def create_json_response(data, status_code):
|
|||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@crawler_splash.route("/crawlers/dashboard", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def crawlers_dashboard():
|
||||
# # TODO: get splash manager status
|
||||
crawler_enabled = crawlers.ping_splash_manager()
|
||||
all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
|
||||
splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
|
||||
date = crawlers.get_current_date()
|
||||
|
||||
return render_template("dashboard_splash_crawler.html", all_splash_crawler_status = all_splash_crawler_status,
|
||||
crawler_enabled=crawler_enabled, date=date,
|
||||
splash_crawlers_latest_stats=splash_crawlers_latest_stats)
|
||||
|
||||
@crawler_splash.route("/crawlers/crawler_dashboard_json", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def crawler_dashboard_json():
|
||||
|
||||
all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
|
||||
splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
|
||||
|
||||
return jsonify({'all_splash_crawler_status': all_splash_crawler_status,
|
||||
'splash_crawlers_latest_stats':splash_crawlers_latest_stats})
|
||||
|
||||
@crawler_splash.route("/crawlers/manual", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
|
|
@ -403,4 +428,11 @@ def crawler_cookiejar_cookie_json_add_post():
|
|||
|
||||
return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid))
|
||||
|
||||
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def crawler_splash_setings():
|
||||
|
||||
return render_template("settings_splash_crawler.html", cookiejar_uuid=True, cookie_uuid=False)
|
||||
|
||||
## - - ##
|
||||
|
|
|
|||
|
|
@ -231,22 +231,22 @@ def delete_auto_crawler(url):
|
|||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@hiddenServices.route("/crawlers/", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def dashboard():
|
||||
crawler_metadata_onion = get_crawler_splash_status('onion')
|
||||
crawler_metadata_regular = get_crawler_splash_status('regular')
|
||||
|
||||
now = datetime.datetime.now()
|
||||
date = now.strftime("%Y%m%d")
|
||||
statDomains_onion = get_stats_last_crawled_domains('onion', date)
|
||||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
|
||||
crawler_enabled=crawler_enabled, date=date,
|
||||
crawler_metadata_regular=crawler_metadata_regular,
|
||||
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
|
||||
# @hiddenServices.route("/crawlers/", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def dashboard():
|
||||
# crawler_metadata_onion = get_crawler_splash_status('onion')
|
||||
# crawler_metadata_regular = get_crawler_splash_status('regular')
|
||||
#
|
||||
# now = datetime.datetime.now()
|
||||
# date = now.strftime("%Y%m%d")
|
||||
# statDomains_onion = get_stats_last_crawled_domains('onion', date)
|
||||
# statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
#
|
||||
# return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
|
||||
# crawler_enabled=crawler_enabled, date=date,
|
||||
# crawler_metadata_regular=crawler_metadata_regular,
|
||||
# statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
|
||||
|
||||
@hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
|
||||
@login_required
|
||||
|
|
@ -439,23 +439,6 @@ def remove_auto_crawler():
|
|||
delete_auto_crawler(url)
|
||||
return redirect(url_for('hiddenServices.auto_crawler', page=page))
|
||||
|
||||
@hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def crawler_dashboard_json():
|
||||
|
||||
crawler_metadata_onion = get_crawler_splash_status('onion')
|
||||
crawler_metadata_regular = get_crawler_splash_status('regular')
|
||||
|
||||
now = datetime.datetime.now()
|
||||
date = now.strftime("%Y%m%d")
|
||||
|
||||
statDomains_onion = get_stats_last_crawled_domains('onion', date)
|
||||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return jsonify({'statDomains_onion': statDomains_onion, 'statDomains_regular': statDomains_regular,
|
||||
'crawler_metadata_onion':crawler_metadata_onion, 'crawler_metadata_regular':crawler_metadata_regular})
|
||||
|
||||
# # TODO: refractor
|
||||
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
|
||||
@login_required
|
||||
|
|
|
|||
|
|
@ -1,267 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'crawler/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</a> UP
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</a> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</a> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_onion_info">
|
||||
{% for crawler in crawler_metadata_onion %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</a> UP
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</a> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</a> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_regular_info">
|
||||
{% for crawler in crawler_metadata_regular %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="d-flex justify-content-center my-4">
|
||||
<div class="card border-secondary" style="max-width: 40rem;">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Show Domain:</h5>
|
||||
<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
|
||||
<div class="input-group mb-3">
|
||||
<input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
|
||||
<div class="input-group-append">
|
||||
<button class="btn btn-info" type="submit" id="btn_show_domain">
|
||||
<i class="fas fa-search"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<hr>
|
||||
<div class="row mb-3">
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
|
||||
<i class="fas fa-user-secret"></i> Onion Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
|
||||
<i class="fab fa-html5"></i> Web Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{% with object_type='domain' %}
|
||||
{% include 'tags/block_obj_tags_search.html' %}
|
||||
{% endwith %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
$("#page-Crawler").addClass("active");
|
||||
$("#nav_dashboard").addClass("active");
|
||||
$( window ).on("focus", function() {
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
$( window ).on("blur", function() {
|
||||
to_refresh = false
|
||||
});
|
||||
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function refresh_crawler_status(){
|
||||
|
||||
$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
|
||||
function(data) {
|
||||
|
||||
$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
|
||||
$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
|
||||
$('#stat_onion_total').text(data.statDomains_onion['total']);
|
||||
$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
|
||||
|
||||
$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
|
||||
$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
|
||||
$('#stat_regular_total').text(data.statDomains_regular['total']);
|
||||
$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
|
||||
|
||||
if(data.crawler_metadata_onion.length!=0){
|
||||
$("#tbody_crawler_onion_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_onion_info');
|
||||
for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
|
||||
var crawler = data.crawler_metadata_onion[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
if(data.crawler_metadata_regular.length!=0){
|
||||
$("#tbody_crawler_regular_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_regular_info');
|
||||
for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
|
||||
var crawler = data.crawler_metadata_regular[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (to_refresh) {
|
||||
setTimeout("refresh_crawler_status()", 10000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
@ -0,0 +1,235 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'crawler/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ splash_crawlers_latest_stats['onion']['domains_up'] }}</a> UP
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ splash_crawlers_latest_stats['onion']['domains_down'] }}</a> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ splash_crawlers_latest_stats['onion']['total'] }}</a> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ splash_crawlers_latest_stats['onion']['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ splash_crawlers_latest_stats['regular']['domains_up'] }}</a> UP
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ splash_crawlers_latest_stats['regular']['domains_down'] }}</a> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ splash_crawlers_latest_stats['regular']['total'] }}</a> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ splash_crawlers_latest_stats['regular']['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_onion_info">
|
||||
{% for splash_crawler in all_splash_crawler_status %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if splash_crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};"></i> {{splash_crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{%if splash_crawler['type']=='onion'%}
|
||||
<i class="fas fa-user-secret"></i>
|
||||
{%else%}
|
||||
<i class="fab fa-html5">
|
||||
{%endif%}
|
||||
</td>
|
||||
<td>
|
||||
{{splash_crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{splash_crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="d-flex justify-content-center my-4">
|
||||
<div class="card border-secondary" style="max-width: 40rem;">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Show Domain:</h5>
|
||||
<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
|
||||
<div class="input-group mb-3">
|
||||
<input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
|
||||
<div class="input-group-append">
|
||||
<button class="btn btn-info" type="submit" id="btn_show_domain">
|
||||
<i class="fas fa-search"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<hr>
|
||||
<div class="row mb-3">
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
|
||||
<i class="fas fa-user-secret"></i> Onion Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
|
||||
<i class="fab fa-html5"></i> Web Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{% with object_type='domain' %}
|
||||
{% include 'tags/block_obj_tags_search.html' %}
|
||||
{% endwith %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
$("#page-Crawler").addClass("active");
|
||||
$("#nav_dashboard").addClass("active");
|
||||
$( window ).on("focus", function() {
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
$( window ).on("blur", function() {
|
||||
to_refresh = false
|
||||
});
|
||||
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function refresh_crawler_status(){
|
||||
|
||||
$.getJSON("{{ url_for('crawler_splash.crawler_dashboard_json') }}",
|
||||
function(data) {
|
||||
|
||||
$('#stat_onion_domain_up').text(data.splash_crawlers_latest_stats['onion']['domains_up']);
|
||||
$('#stat_onion_domain_down').text(data.splash_crawlers_latest_stats['onion']['domains_down']);
|
||||
$('#stat_onion_total').text(data.splash_crawlers_latest_stats['onion']['total']);
|
||||
$('#stat_onion_queue').text(data.splash_crawlers_latest_stats['onion']['domains_queue']);
|
||||
|
||||
$('#stat_regular_domain_up').text(data.splash_crawlers_latest_stats['regular']['domains_up']);
|
||||
$('#stat_regular_domain_down').text(data.splash_crawlers_latest_stats['regular']['domains_down']);
|
||||
$('#stat_regular_total').text(data.splash_crawlers_latest_stats['regular']['total']);
|
||||
$('#stat_regular_queue').text(data.splash_crawlers_latest_stats['regular']['domains_queue']);
|
||||
|
||||
if(data.all_splash_crawler_status.length!=0){
|
||||
$("#tbody_crawler_onion_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_onion_info');
|
||||
for (var i = 0; i < data.all_splash_crawler_status.length; i++) {
|
||||
var crawler = data.all_splash_crawler_status[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
if(crawler['type'] === 'onion'){
|
||||
icon_t = 'fas fa-user-secret';
|
||||
} else {
|
||||
icon_t = 'fab fa-html5';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
var newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td><i class=\""+icon_t+"\"></i></td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(3);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (to_refresh) {
|
||||
setTimeout("refresh_crawler_status()", 10000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hiddenServices.dashboard')}}" id="nav_dashboard">
|
||||
<a class="nav-link" href="{{url_for('crawler_splash.crawlers_dashboard')}}" id="nav_dashboard">
|
||||
<i class="fas fa-search"></i>
|
||||
<span>Dashboard</span>
|
||||
</a>
|
||||
|
|
@ -43,6 +43,12 @@
|
|||
Automatic Crawler
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hiddenServices.auto_crawler')}}" id="nav_settings">
|
||||
<i class="fas fa-cog"></i>
|
||||
Settings
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_domains_explorer">
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
||||
<a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
|
||||
|
|
|
|||
Loading…
Reference in a new issue