2020-08-17 19:52:57 +00:00
|
|
|
<!DOCTYPE html>
|
|
|
|
|
|
|
|
|
|
<html>
|
|
|
|
|
<head>
|
|
|
|
|
<title>AIL-Framework</title>
|
|
|
|
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
|
|
|
|
<!-- Core CSS -->
|
|
|
|
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
|
|
|
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
2022-10-25 14:25:19 +00:00
|
|
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
<!-- JS -->
|
|
|
|
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
|
|
|
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
2022-10-25 14:25:19 +00:00
|
|
|
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
|
|
|
|
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
|
<body>
|
|
|
|
|
|
|
|
|
|
{% include 'nav_bar.html' %}
|
|
|
|
|
|
|
|
|
|
<div class="container-fluid">
|
|
|
|
|
<div class="row">
|
|
|
|
|
|
|
|
|
|
{% include 'crawler/menu_sidebar.html' %}
|
|
|
|
|
|
|
|
|
|
<div class="col-12 col-lg-10" id="core_content">
|
|
|
|
|
|
|
|
|
|
{% include 'crawler/crawler_disabled.html' %}
|
|
|
|
|
|
|
|
|
|
<div class="row">
|
|
|
|
|
<div class="col-xl-6">
|
|
|
|
|
|
|
|
|
|
<div class="card mt-1 mb-1">
|
|
|
|
|
<div class="card-header text-white bg-dark">
|
2022-10-25 14:25:19 +00:00
|
|
|
<h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
|
2020-08-17 19:52:57 +00:00
|
|
|
<div class="row">
|
|
|
|
|
<div class="col-6">
|
2024-09-23 14:43:15 +00:00
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ crawlers_latest_stats['onion']['up'] }}</a> UP
|
|
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ crawlers_latest_stats['onion']['down'] }}</a> DOWN
|
2020-08-17 19:52:57 +00:00
|
|
|
</div>
|
|
|
|
|
<div class="col-6">
|
2023-04-24 11:35:55 +00:00
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&up=True&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ crawlers_latest_stats['onion']['crawled'] }}</a> Crawled
|
2022-10-25 14:25:19 +00:00
|
|
|
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ crawlers_latest_stats['onion']['queue'] }}</span> Queue
|
2020-08-17 19:52:57 +00:00
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
<div class="col-xl-6">
|
|
|
|
|
<div class="card mt-1 mb-1">
|
|
|
|
|
<div class="card-header text-white bg-dark">
|
2022-10-25 14:25:19 +00:00
|
|
|
<h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=web"><i class="fab fa-html5"></i> Web Crawlers</a></h5>
|
2020-08-17 19:52:57 +00:00
|
|
|
<div class="row">
|
|
|
|
|
<div class="col-6">
|
2024-09-23 14:43:15 +00:00
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_domain_up">{{ crawlers_latest_stats['web']['up'] }}</a> UP
|
|
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_web_domain_down">{{ crawlers_latest_stats['web']['down'] }}</a> DOWN
|
2020-08-17 19:52:57 +00:00
|
|
|
</div>
|
|
|
|
|
<div class="col-6">
|
2023-04-24 11:35:55 +00:00
|
|
|
<a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&up=True&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_total">{{ crawlers_latest_stats['web']['crawled'] }}</a> Crawled
|
2022-10-25 14:25:19 +00:00
|
|
|
<span class="badge badge-warning ml-md-3" id="stat_web_queue">{{ crawlers_latest_stats['web']['queue'] }}</span> Queue
|
2020-08-17 19:52:57 +00:00
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<table class="table">
|
|
|
|
|
<tbody id="tbody_crawler_onion_info">
|
2022-10-25 14:25:19 +00:00
|
|
|
{% for crawler in crawlers_status %}
|
2020-08-17 19:52:57 +00:00
|
|
|
<tr>
|
|
|
|
|
<td>
|
2022-10-25 14:25:19 +00:00
|
|
|
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['start_time']}}
|
2020-08-17 19:52:57 +00:00
|
|
|
</td>
|
|
|
|
|
<td>
|
2022-10-25 14:25:19 +00:00
|
|
|
{%if crawler['type']=='onion'%}
|
2020-08-17 19:52:57 +00:00
|
|
|
<i class="fas fa-user-secret"></i>
|
|
|
|
|
{%else%}
|
|
|
|
|
<i class="fab fa-html5">
|
|
|
|
|
{%endif%}
|
|
|
|
|
</td>
|
|
|
|
|
<td>
|
2022-10-25 14:25:19 +00:00
|
|
|
{{crawler['domain']}}
|
2020-08-17 19:52:57 +00:00
|
|
|
</td>
|
2022-10-25 14:25:19 +00:00
|
|
|
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
|
|
|
|
{{crawler['status']}}
|
2020-08-17 19:52:57 +00:00
|
|
|
</td>
|
|
|
|
|
</tr>
|
|
|
|
|
{% endfor %}
|
|
|
|
|
</tbody>
|
|
|
|
|
</table>
|
|
|
|
|
|
2021-02-10 14:50:48 +00:00
|
|
|
{% include 'domains/block_domains_name_search.html' %}
|
2022-10-25 14:25:19 +00:00
|
|
|
<div class="d-flex justify-content-center my-4">
|
|
|
|
|
{% include 'crawler/show_domains_by_daterange.html' %}
|
|
|
|
|
</div>
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
<hr>
|
|
|
|
|
<div class="row mb-3">
|
|
|
|
|
<div class="col-xl-6">
|
|
|
|
|
<div class="text-center">
|
|
|
|
|
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
|
|
|
|
|
<i class="fas fa-user-secret"></i> Onion Domain Explorer
|
|
|
|
|
</a>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="col-xl-6">
|
|
|
|
|
<div class="text-center">
|
|
|
|
|
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
|
|
|
|
|
<i class="fab fa-html5"></i> Web Domain Explorer
|
|
|
|
|
</a>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
{% with object_type='domain' %}
|
|
|
|
|
{% include 'tags/block_obj_tags_search.html' %}
|
|
|
|
|
{% endwith %}
|
|
|
|
|
|
2020-12-11 20:02:07 +00:00
|
|
|
{% with object_type='domain' %}
|
|
|
|
|
{% include 'domains/block_languages_search.html' %}
|
|
|
|
|
{% endwith %}
|
|
|
|
|
|
2020-08-17 19:52:57 +00:00
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</body>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
|
|
|
var to_refresh = false
|
|
|
|
|
$(document).ready(function(){
|
|
|
|
|
$("#page-Crawler").addClass("active");
|
|
|
|
|
$("#nav_dashboard").addClass("active");
|
2022-10-25 14:25:19 +00:00
|
|
|
$('#date-range-from').dateRangePicker({
|
|
|
|
|
separator : ' to ',
|
|
|
|
|
getValue: function(){
|
|
|
|
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
|
|
|
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
|
|
|
|
else
|
|
|
|
|
return '';
|
|
|
|
|
},
|
|
|
|
|
setValue: function(s,s1,s2){
|
|
|
|
|
$('#date-range-from-input').val(s1);
|
|
|
|
|
$('#date-range-to-input').val(s2);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
$('#date-range-to').dateRangePicker({
|
|
|
|
|
separator : ' to ',
|
|
|
|
|
getValue: function(){
|
|
|
|
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
|
|
|
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
|
|
|
|
else
|
|
|
|
|
return '';
|
|
|
|
|
},
|
|
|
|
|
setValue: function(s,s1,s2){
|
|
|
|
|
$('#date-range-from-input').val(s1);
|
|
|
|
|
$('#date-range-to-input').val(s2);
|
|
|
|
|
}
|
|
|
|
|
});
|
2020-08-17 19:52:57 +00:00
|
|
|
$( window ).on("focus", function() {
|
|
|
|
|
to_refresh = true
|
|
|
|
|
refresh_crawler_status();
|
|
|
|
|
});
|
|
|
|
|
$( window ).on("blur", function() {
|
|
|
|
|
to_refresh = false
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
to_refresh = true
|
|
|
|
|
refresh_crawler_status();
|
2022-10-25 14:25:19 +00:00
|
|
|
|
2020-08-17 19:52:57 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
function toggle_sidebar(){
|
|
|
|
|
if($('#nav_menu').is(':visible')){
|
|
|
|
|
$('#nav_menu').hide();
|
|
|
|
|
$('#side_menu').removeClass('border-right')
|
|
|
|
|
$('#side_menu').removeClass('col-lg-2')
|
|
|
|
|
$('#core_content').removeClass('col-lg-10')
|
|
|
|
|
}else{
|
|
|
|
|
$('#nav_menu').show();
|
|
|
|
|
$('#side_menu').addClass('border-right')
|
|
|
|
|
$('#side_menu').addClass('col-lg-2')
|
|
|
|
|
$('#core_content').addClass('col-lg-10')
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function refresh_crawler_status(){
|
|
|
|
|
|
|
|
|
|
$.getJSON("{{ url_for('crawler_splash.crawler_dashboard_json') }}",
|
|
|
|
|
function(data) {
|
|
|
|
|
|
2022-10-25 14:25:19 +00:00
|
|
|
$('#stat_onion_domain_up').text(data.stats['onion']['up']);
|
|
|
|
|
$('#stat_onion_domain_down').text(data.stats['onion']['down']);
|
|
|
|
|
$('#stat_onion_total').text(data.stats['onion']['crawled']);
|
|
|
|
|
$('#stat_onion_queue').text(data.stats['onion']['queue']);
|
2020-08-17 19:52:57 +00:00
|
|
|
|
2022-10-25 14:25:19 +00:00
|
|
|
$('#stat_web_domain_up').text(data.stats['web']['up']);
|
|
|
|
|
$('#stat_web_domain_down').text(data.stats['web']['down']);
|
|
|
|
|
$('#stat_web_total').text(data.stats['web']['crawled']);
|
|
|
|
|
$('#stat_web_queue').text(data.stats['web']['queue']);
|
2020-08-17 19:52:57 +00:00
|
|
|
|
2023-03-14 16:36:42 +00:00
|
|
|
$("#tbody_crawler_onion_info").empty();
|
|
|
|
|
|
2022-10-25 14:25:19 +00:00
|
|
|
if(data.crawlers_status.length!=0){
|
2020-08-17 19:52:57 +00:00
|
|
|
var tableRef = document.getElementById('tbody_crawler_onion_info');
|
2022-10-25 14:25:19 +00:00
|
|
|
for (var i = 0; i < data.crawlers_status.length; i++) {
|
|
|
|
|
var crawler = data.crawlers_status[i];
|
2020-08-17 19:52:57 +00:00
|
|
|
var newRow = tableRef.insertRow(tableRef.rows.length);
|
|
|
|
|
var text_color;
|
|
|
|
|
var icon;
|
|
|
|
|
if(crawler['status']){
|
|
|
|
|
text_color = 'Green';
|
|
|
|
|
icon = 'check';
|
|
|
|
|
} else {
|
|
|
|
|
text_color = 'Red';
|
|
|
|
|
icon = 'times';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if(crawler['type'] === 'onion'){
|
|
|
|
|
icon_t = 'fas fa-user-secret';
|
|
|
|
|
} else {
|
|
|
|
|
icon_t = 'fab fa-html5';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var newCell = newRow.insertCell(0);
|
2022-10-25 14:25:19 +00:00
|
|
|
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['start_time']+"</td>";
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
var newCell = newRow.insertCell(1);
|
|
|
|
|
newCell.innerHTML = "<td><i class=\""+icon_t+"\"></i></td>";
|
|
|
|
|
|
|
|
|
|
newCell = newRow.insertCell(2);
|
2022-10-25 14:25:19 +00:00
|
|
|
newCell.innerHTML = "<td>"+crawler['domain']+"</td>";
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
newCell = newRow.insertCell(3);
|
2022-10-25 14:25:19 +00:00
|
|
|
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status']+"</div></td>";
|
2020-08-17 19:52:57 +00:00
|
|
|
|
|
|
|
|
//$("#panel_crawler").show();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
if (to_refresh) {
|
|
|
|
|
setTimeout("refresh_crawler_status()", 10000);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
</script>
|
