mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [crawler UI] bootstrap 4 migration: show domains by daterange + bug fix hidden services tags
This commit is contained in:
Terrtia
parent
ff14fe6279
commit
a183695ddb
5 changed files with 252 additions and 338 deletions
|
|
@ -105,18 +105,20 @@ class HiddenServices(object):
|
||||||
return self.tags
|
return self.tags
|
||||||
|
|
||||||
def update_domain_tags(self, item):
|
def update_domain_tags(self, item):
|
||||||
if self.r_serv_metadata.exists('tag:{}'.format(item)):
|
if item:
|
||||||
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(item))
|
|
||||||
# update path here
|
if self.r_serv_metadata.exists('tag:{}'.format(item)):
|
||||||
else:
|
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(item))
|
||||||
# need to remove it
|
# update path here
|
||||||
if self.paste_directory in item:
|
|
||||||
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(item.replace(self.paste_directory+'/', '')))
|
|
||||||
# need to remove it
|
|
||||||
else:
|
else:
|
||||||
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(os.path.join(self.paste_directory, item)))
|
# need to remove it
|
||||||
for tag in p_tags:
|
if self.paste_directory in item:
|
||||||
self.tags[tag] = self.tags.get(tag, 0) + 1
|
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(item.replace(self.paste_directory+'/', '')))
|
||||||
|
# need to remove it
|
||||||
|
else:
|
||||||
|
p_tags = self.r_serv_metadata.smembers('tag:{}'.format(os.path.join(self.paste_directory, item)))
|
||||||
|
for tag in p_tags:
|
||||||
|
self.tags[tag] = self.tags.get(tag, 0) + 1
|
||||||
|
|
||||||
def get_first_crawled(self):
|
def get_first_crawled(self):
|
||||||
res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
|
res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
|
||||||
|
|
@ -150,7 +152,9 @@ class HiddenServices(object):
|
||||||
#todo use the right paste
|
#todo use the right paste
|
||||||
def get_last_crawled_pastes(self, item_root=None):
|
def get_last_crawled_pastes(self, item_root=None):
|
||||||
if item_root is None:
|
if item_root is None:
|
||||||
item_root = self.get_domain_crawled_core_item(self)
|
item_root = self.get_domain_crawled_core_item()
|
||||||
|
if item_root:
|
||||||
|
item_root = item_root['root_item']
|
||||||
return self.get_all_pastes_domain(item_root)
|
return self.get_all_pastes_domain(item_root)
|
||||||
|
|
||||||
def get_all_pastes_domain(self, root_item):
|
def get_all_pastes_domain(self, root_item):
|
||||||
|
|
|
||||||
|
|
@ -82,6 +82,13 @@ def is_valid_domain(domain):
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def is_valid_service_type(service_type):
|
||||||
|
accepted_service = ['onion', 'regular']
|
||||||
|
if service_type in accepted_service:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
def get_onion_status(domain, date):
|
def get_onion_status(domain, date):
|
||||||
if r_serv_onion.sismember('onion_up:'+date , domain):
|
if r_serv_onion.sismember('onion_up:'+date , domain):
|
||||||
return True
|
return True
|
||||||
|
|
@ -114,13 +121,17 @@ def get_domain_from_url(url):
|
||||||
def get_last_domains_crawled(type):
|
def get_last_domains_crawled(type):
|
||||||
return r_serv_onion.lrange('last_{}'.format(type), 0 ,-1)
|
return r_serv_onion.lrange('last_{}'.format(type), 0 ,-1)
|
||||||
|
|
||||||
|
def get_nb_domains_inqueue(type):
|
||||||
|
nb = r_serv_onion.scard('{}_crawler_queue'.format(type))
|
||||||
|
nb += r_serv_onion.scard('{}_crawler_priority_queue'.format(type))
|
||||||
|
return nb
|
||||||
|
|
||||||
def get_stats_last_crawled_domains(type, date):
|
def get_stats_last_crawled_domains(type, date):
|
||||||
statDomains = {}
|
statDomains = {}
|
||||||
statDomains['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
statDomains['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
||||||
statDomains['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(type, date))
|
statDomains['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(type, date))
|
||||||
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
||||||
statDomains['domains_queue'] = r_serv_onion.scard('{}_crawler_queue'.format(type))
|
statDomains['domains_queue'] = get_nb_domains_inqueue(type)
|
||||||
statDomains['domains_queue'] += r_serv_onion.scard('{}_crawler_priority_queue'.format(type))
|
|
||||||
return statDomains
|
return statDomains
|
||||||
|
|
||||||
def get_last_crawled_domains_metadata(list_domains_crawled, date, type=None, auto_mode=False):
|
def get_last_crawled_domains_metadata(list_domains_crawled, date, type=None, auto_mode=False):
|
||||||
|
|
@ -561,20 +572,28 @@ def last_crawled_domains_with_stats_json():
|
||||||
def get_onions_by_daterange():
|
def get_onions_by_daterange():
|
||||||
date_from = request.form.get('date_from')
|
date_from = request.form.get('date_from')
|
||||||
date_to = request.form.get('date_to')
|
date_to = request.form.get('date_to')
|
||||||
|
service_type = request.form.get('service_type')
|
||||||
domains_up = request.form.get('domains_up')
|
domains_up = request.form.get('domains_up')
|
||||||
domains_down = request.form.get('domains_down')
|
domains_down = request.form.get('domains_down')
|
||||||
domains_tags = request.form.get('domains_tags')
|
domains_tags = request.form.get('domains_tags')
|
||||||
|
|
||||||
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
|
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, service_type=service_type, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
|
||||||
|
|
||||||
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
|
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
|
||||||
def show_domains_by_daterange():
|
def show_domains_by_daterange():
|
||||||
date_from = request.args.get('date_from')
|
date_from = request.args.get('date_from')
|
||||||
date_to = request.args.get('date_to')
|
date_to = request.args.get('date_to')
|
||||||
|
service_type = request.args.get('service_type')
|
||||||
domains_up = request.args.get('domains_up')
|
domains_up = request.args.get('domains_up')
|
||||||
domains_down = request.args.get('domains_down')
|
domains_down = request.args.get('domains_down')
|
||||||
domains_tags = request.args.get('domains_tags')
|
domains_tags = request.args.get('domains_tags')
|
||||||
|
|
||||||
|
# incorrect service type
|
||||||
|
if not is_valid_service_type(service_type):
|
||||||
|
service_type = 'onion'
|
||||||
|
|
||||||
|
type_name = dic_type_name[service_type]
|
||||||
|
|
||||||
date_range = []
|
date_range = []
|
||||||
if date_from is not None and date_to is not None:
|
if date_from is not None and date_to is not None:
|
||||||
#change format
|
#change format
|
||||||
|
|
@ -595,12 +614,22 @@ def show_domains_by_daterange():
|
||||||
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
||||||
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
||||||
|
|
||||||
|
statDomains = {}
|
||||||
|
statDomains['domains_up'] = 0
|
||||||
|
statDomains['domains_down'] = 0
|
||||||
|
statDomains['total'] = 0
|
||||||
|
statDomains['domains_queue'] = get_nb_domains_inqueue(service_type)
|
||||||
|
|
||||||
domains_by_day = {}
|
domains_by_day = {}
|
||||||
domain_metadata = {}
|
domain_metadata = {}
|
||||||
|
stats_by_date = {}
|
||||||
for date in date_range:
|
for date in date_range:
|
||||||
|
stats_by_date[date] = {}
|
||||||
|
stats_by_date[date]['domain_up'] = 0
|
||||||
|
stats_by_date[date]['domain_down'] = 0
|
||||||
if domains_up:
|
if domains_up:
|
||||||
domains_up = True
|
domains_up = True
|
||||||
domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
|
domains_by_day[date] = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date)))
|
||||||
for domain in domains_by_day[date]:
|
for domain in domains_by_day[date]:
|
||||||
h = HiddenServices(domain, 'onion')
|
h = HiddenServices(domain, 'onion')
|
||||||
domain_metadata[domain] = {}
|
domain_metadata[domain] = {}
|
||||||
|
|
@ -608,19 +637,21 @@ def show_domains_by_daterange():
|
||||||
domains_tags = True
|
domains_tags = True
|
||||||
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
|
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
|
||||||
|
|
||||||
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
domain_metadata[domain]['last_check'] = r_serv_onion.hget('{}_metadata:{}'.format(service_type, domain), 'last_check')
|
||||||
if domain_metadata[domain]['last_check'] is None:
|
if domain_metadata[domain]['last_check'] is None:
|
||||||
domain_metadata[domain]['last_check'] = '********'
|
domain_metadata[domain]['last_check'] = '********'
|
||||||
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('{}_metadata:{}'.format(service_type, domain), 'first_seen')
|
||||||
if domain_metadata[domain]['first_seen'] is None:
|
if domain_metadata[domain]['first_seen'] is None:
|
||||||
domain_metadata[domain]['first_seen'] = '********'
|
domain_metadata[domain]['first_seen'] = '********'
|
||||||
domain_metadata[domain]['status_text'] = 'UP'
|
domain_metadata[domain]['status_text'] = 'UP'
|
||||||
domain_metadata[domain]['status_color'] = 'Green'
|
domain_metadata[domain]['status_color'] = 'Green'
|
||||||
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
|
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
|
||||||
|
statDomains['domains_up'] += 1
|
||||||
|
stats_by_date[date]['domain_up'] += 1
|
||||||
|
|
||||||
if domains_down:
|
if domains_down:
|
||||||
domains_down = True
|
domains_down = True
|
||||||
domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
|
domains_by_day_down = list(r_serv_onion.smembers('{}_down:{}'.format(service_type, date)))
|
||||||
if domains_up:
|
if domains_up:
|
||||||
domains_by_day[date].extend(domains_by_day_down)
|
domains_by_day[date].extend(domains_by_day_down)
|
||||||
else:
|
else:
|
||||||
|
|
@ -630,20 +661,27 @@ def show_domains_by_daterange():
|
||||||
domain_metadata[domain] = {}
|
domain_metadata[domain] = {}
|
||||||
#domain_metadata[domain]['tags'] = h.get_domain_tags()
|
#domain_metadata[domain]['tags'] = h.get_domain_tags()
|
||||||
|
|
||||||
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
domain_metadata[domain]['last_check'] = r_serv_onion.hget('{}_metadata:{}'.format(service_type, domain), 'last_check')
|
||||||
if domain_metadata[domain]['last_check'] is None:
|
if domain_metadata[domain]['last_check'] is None:
|
||||||
domain_metadata[domain]['last_check'] = '********'
|
domain_metadata[domain]['last_check'] = '********'
|
||||||
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('{}_metadata:{}'.format(service_type, domain), 'first_seen')
|
||||||
if domain_metadata[domain]['first_seen'] is None:
|
if domain_metadata[domain]['first_seen'] is None:
|
||||||
domain_metadata[domain]['first_seen'] = '********'
|
domain_metadata[domain]['first_seen'] = '********'
|
||||||
|
|
||||||
domain_metadata[domain]['status_text'] = 'DOWN'
|
domain_metadata[domain]['status_text'] = 'DOWN'
|
||||||
domain_metadata[domain]['status_color'] = 'Red'
|
domain_metadata[domain]['status_color'] = 'Red'
|
||||||
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
|
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
|
||||||
|
statDomains['domains_down'] += 1
|
||||||
|
stats_by_date[date]['domain_down'] += 1
|
||||||
|
|
||||||
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
|
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
||||||
|
|
||||||
|
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day,
|
||||||
|
statDomains=statDomains, type_name=type_name,
|
||||||
|
domain_metadata=domain_metadata,
|
||||||
|
stats_by_date=stats_by_date,
|
||||||
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
|
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
|
||||||
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
|
domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
@hiddenServices.route("/crawlers/show_domain", methods=['GET'])
|
@hiddenServices.route("/crawlers/show_domain", methods=['GET'])
|
||||||
def show_domain():
|
def show_domain():
|
||||||
|
|
|
||||||
|
|
@ -85,62 +85,7 @@
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-xl-6">
|
<div class="col-12 col-xl-6">
|
||||||
|
{% include 'crawler/show_domains_by_daterange.html' %}
|
||||||
<div class="card text-white bg-dark mb-3 mt-1">
|
|
||||||
<div class="card-header">
|
|
||||||
<div class="row">
|
|
||||||
<div class="col-6">
|
|
||||||
<span class="badge badge-success">{{ statDomains['domains_up'] }}</span> UP
|
|
||||||
<span class="badge badge-danger ml-md-3">{{ statDomains['domains_down'] }}</span> DOWN
|
|
||||||
</div>
|
|
||||||
<div class="col-6">
|
|
||||||
<span class="badge badge-success">{{ statDomains['total'] }}</span> Crawled
|
|
||||||
<span class="badge badge-warning ml-md-3">{{ statDomains['domains_queue'] }}</span> Queue
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="card-body">
|
|
||||||
<h5 class="card-title">Select domains by date range :</h5>
|
|
||||||
<p class="card-text">Some quick example text to build on the card title and make up the bulk of the card's content.</p>
|
|
||||||
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
|
|
||||||
<div class="row">
|
|
||||||
<div class="col-6">
|
|
||||||
<div class="input-group" id="date-range-from">
|
|
||||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
|
||||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
|
||||||
</div>
|
|
||||||
<div class="input-group" id="date-range-to">
|
|
||||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
|
||||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="col-6">
|
|
||||||
<div class="custom-control custom-switch">
|
|
||||||
<input class="custom-control-input" type="checkbox" name="domains_up" value="True" id="domains_up_id" checked>
|
|
||||||
<label class="custom-control-label" for="domains_up_id">
|
|
||||||
<span class="badge badge-success"><i class="fas fa-check-circle"></i> Domains UP </span>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
<div class="custom-control custom-switch">
|
|
||||||
<input class="custom-control-input" type="checkbox" name="domains_down" value="True" id="domains_down_id">
|
|
||||||
<label class="custom-control-label" for="domains_down_id">
|
|
||||||
<span class="badge badge-danger"><i class="fas fa-times-circle"></i> Domains DOWN</span>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
<div class="custom-control custom-switch mt-2">
|
|
||||||
<input class="custom-control-input" type="checkbox" name="domains_tags" value="True" id="domains_tags_id">
|
|
||||||
<label class="custom-control-label" for="domains_tags_id">
|
|
||||||
<span class="badge badge-dark"><i class="fas fa-tags"></i> Domains Tags</span>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<button class="btn btn-primary">
|
|
||||||
<i class="fas fa-eye"></i> Show {{type_name}}s
|
|
||||||
</button>
|
|
||||||
<form>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div id="barchart_type">
|
<div id="barchart_type">
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1,291 +1,162 @@
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
|
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8">
|
<meta charset="utf-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<title>AIL-Framework</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
<title>Hidden Service - AIL</title>
|
<!-- JS -->
|
||||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||||
|
|
||||||
<!-- Core CSS -->
|
</head>
|
||||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
<body>
|
||||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
|
||||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
|
||||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
|
||||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
|
||||||
<!-- JS -->
|
|
||||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
|
||||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
|
||||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
|
||||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
|
||||||
|
|
||||||
<style>
|
{% include 'nav_bar.html' %}
|
||||||
.red_table thead{
|
|
||||||
background: #d91f2d;
|
|
||||||
color: #fff;
|
|
||||||
}
|
|
||||||
.input-group .form-control {
|
|
||||||
position: unset;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
|
|
||||||
</head>
|
<div class="container-fluid">
|
||||||
<body>
|
<div class="row">
|
||||||
|
|
||||||
{% include 'navbar.html' %}
|
{% include 'crawler/menu_sidebar.html' %}
|
||||||
|
|
||||||
<div id="page-wrapper">
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-md-8">
|
<div class="col-12 col-xl-6">
|
||||||
|
|
||||||
{% for date in date_range %}
|
{% for date in date_range %}
|
||||||
{% if domains_by_day[date]%}
|
{% if domains_by_day[date]%}
|
||||||
<div class="panel panel-info" style="text-align:center;">
|
<div class="card text-center mt-1 mb-3">
|
||||||
<div class="panel-heading">
|
<div class="card-header bg-dark text-white">
|
||||||
<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<table id="table_{{date}}" class="red_table table table-striped table-bordered">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Domain</th>
|
|
||||||
<th>First Seen</th>
|
|
||||||
<th>Last Check</th>
|
|
||||||
<th>Status</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{% for domain in domains_by_day[date] %}
|
|
||||||
<tr>
|
|
||||||
<td>
|
|
||||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ domain }}">{{ domain }}</a>
|
|
||||||
<div>
|
|
||||||
{% for tag in domain_metadata[domain]['tags'] %}
|
|
||||||
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
|
|
||||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
|
|
||||||
</a>
|
|
||||||
{% endfor %}
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
|
|
||||||
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
|
|
||||||
<td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
|
|
||||||
<i class="fa {{domain_metadata[domain]['status_icon']}} fa-2x"></i>
|
|
||||||
{{domain_metadata[domain]['status_text']}}
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="col-md-4">
|
|
||||||
<div class="panel panel-info" style="text-align:center;">
|
|
||||||
<div class="panel-heading">
|
|
||||||
Select domains by date range :
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
|
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-md-6">
|
<div class="col-6">
|
||||||
<div class="input-group">
|
<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
|
||||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
|
||||||
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
|
|
||||||
</div>
|
|
||||||
<div class="input-group">
|
|
||||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
|
||||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
|
||||||
</div>
|
|
||||||
<div class="checkbox">
|
|
||||||
<label>
|
|
||||||
<input type="checkbox" name="domains_tags" value="True" {% if domains_tags %}checked{% endif %}>
|
|
||||||
<div style="color:#286090; display:inline-block">
|
|
||||||
Domains Tags <i class="fa fa-tags"></i>
|
|
||||||
</div>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-6">
|
||||||
<div class="checkbox">
|
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type={{type}}&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success">{{ stats_by_date[date]['domain_up'] }}</a> UP
|
||||||
<label>
|
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type={{type}}&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3">{{ stats_by_date[date]['domain_down'] }}</a> DOWN
|
||||||
<input type="checkbox" name="domains_up" value="True" {% if domains_up %}checked{% endif %}>
|
|
||||||
<div style="color:Green; display:inline-block">
|
|
||||||
Domains UP <i class="fa fa-check-circle"></i>
|
|
||||||
</div>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
<div class="checkbox">
|
|
||||||
<label>
|
|
||||||
<input type="checkbox" name="domains_down" value="True" {% if domains_down %}checked{% endif %}>
|
|
||||||
<div style="color:Red; display:inline-block">
|
|
||||||
Domains DOWN <i class="fa fa-times-circle"></i>
|
|
||||||
</div>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button class="btn btn-primary" style="text-align:center;">
|
</div>
|
||||||
<i class="fa fa-eye-slash"></i> Show Onions
|
<div class="card-body px-1">
|
||||||
</button>
|
|
||||||
<form>
|
<table id="table_{{date}}" class="table table-striped table-bordered">
|
||||||
|
<thead class="bg-dark text-white">
|
||||||
|
<tr>
|
||||||
|
<th>Domain</th>
|
||||||
|
<th>First Seen</th>
|
||||||
|
<th>Last Check</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for domain in domains_by_day[date] %}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ domain }}">{{ domain }}</a>
|
||||||
|
<div>
|
||||||
|
{% for tag in domain_metadata[domain]['tags'] %}
|
||||||
|
<a href="{{ url_for('Tags.Tags_page') }}?date_from={{date}}&date_to={{date}}<ags={{ tag }}">
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
|
||||||
|
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
|
||||||
|
<td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
|
||||||
|
<i class="fas {{domain_metadata[domain]['status_icon']}}"></i>
|
||||||
|
{{domain_metadata[domain]['status_text']}}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
{% include 'crawler/show_domains_by_daterange.html' %}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<!-- /#page-wrapper -->
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script>
|
</body>
|
||||||
var all_graph = {};
|
|
||||||
$(document).ready(function(){
|
|
||||||
activePage = "page-hiddenServices"
|
|
||||||
$("#"+activePage).addClass("active");
|
|
||||||
|
|
||||||
$('#date-range-from').dateRangePicker({
|
|
||||||
separator : ' to ',
|
|
||||||
getValue: function()
|
|
||||||
{
|
|
||||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
|
||||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
|
||||||
else
|
|
||||||
return '';
|
|
||||||
},
|
|
||||||
setValue: function(s,s1,s2)
|
|
||||||
{
|
|
||||||
$('#date-range-from').val(s1);
|
|
||||||
$('#date-range-to').val(s2);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
$('#date-range-to').dateRangePicker({
|
|
||||||
separator : ' to ',
|
|
||||||
getValue: function()
|
|
||||||
{
|
|
||||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
|
||||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
|
||||||
else
|
|
||||||
return '';
|
|
||||||
},
|
|
||||||
setValue: function(s,s1,s2)
|
|
||||||
{
|
|
||||||
$('#date-range-from').val(s1);
|
|
||||||
$('#date-range-to').val(s2);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
{% for date in date_range %}
|
|
||||||
$('#table_{{date}}').DataTable({
|
|
||||||
"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
|
|
||||||
"iDisplayLength": 15,
|
|
||||||
"order": [[ 0, "asc" ]]
|
|
||||||
});
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
//all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
|
||||||
});
|
|
||||||
/*$(window).on("resize", function() {
|
|
||||||
all_graph.onResize();
|
|
||||||
});*/
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
/*
|
$(document).ready(function(){
|
||||||
all_graph.onResize = function () {
|
$("#page-Crawler").addClass("active");
|
||||||
var aspect = 1000 / 500, all_graph = $("#graph_div");
|
|
||||||
var targetWidth = all_graph.parent().width();
|
|
||||||
all_graph.attr("width", targetWidth);
|
|
||||||
all_graph.attr("height", targetWidth / aspect);
|
|
||||||
}
|
|
||||||
window.all_graph = all_graph;
|
|
||||||
|
|
||||||
function create_line_chart(id, url){
|
$('#date-range-from').dateRangePicker({
|
||||||
var width = 650;
|
separator : ' to ',
|
||||||
var height = Math.round(width / 6);
|
getValue: function(){
|
||||||
var margin = {top: 20, right: 55, bottom: 50, left: 40};
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
var x = d3.scaleTime().range([0, width]);
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
else
|
||||||
var xAxis = d3.axisBottom(x);
|
return '';
|
||||||
var yAxis = d3.axisLeft(y);
|
},
|
||||||
var parseTime = d3.timeParse("%Y-%m-%d");
|
setValue: function(s,s1,s2){
|
||||||
var line = d3.line()
|
$('#date-range-from-input').val(s1);
|
||||||
.x(function(d) {
|
$('#date-range-to-input').val(s2);
|
||||||
return x(d.date);
|
}
|
||||||
})
|
|
||||||
.y(function(d) {
|
|
||||||
return y(d.value);
|
|
||||||
});
|
|
||||||
var svg_line = d3.select('#'+id).append('svg')
|
|
||||||
.attr("id", "graph_div")
|
|
||||||
.attr("width", width + margin.left + margin.right)
|
|
||||||
.attr("height", height + margin.top + margin.bottom)
|
|
||||||
.append('g')
|
|
||||||
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
|
|
||||||
var div = d3.select('body').append('div')
|
|
||||||
.attr('class', 'tooltip')
|
|
||||||
.style('opacity', 0);
|
|
||||||
//add div tooltip
|
|
||||||
d3.json(url)
|
|
||||||
.then(function(data){
|
|
||||||
data.forEach(function(d) {
|
|
||||||
d.date_label = d.date;
|
|
||||||
d.date = parseTime(d.date);
|
|
||||||
d.value = +d.value;
|
|
||||||
});
|
|
||||||
// fit the data
|
|
||||||
x.domain(d3.extent(data, function(d) { return d.date; }));
|
|
||||||
//x.domain(data.map(function (d) { return d.date; })); //E
|
|
||||||
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
|
|
||||||
//line
|
|
||||||
svg_line.append("path")
|
|
||||||
.data([data])
|
|
||||||
.attr("class", "line_graph")
|
|
||||||
.attr("d", line);
|
|
||||||
// add X axis
|
|
||||||
svg_line.append("g")
|
|
||||||
.attr("transform", "translate(0," + height + ")")
|
|
||||||
.call(d3.axisBottom(x))
|
|
||||||
.selectAll("text")
|
|
||||||
.style("text-anchor", "end")
|
|
||||||
.attr("transform", "rotate(-45)" );
|
|
||||||
// Add the Y Axis
|
|
||||||
svg_line.append("g")
|
|
||||||
.call(d3.axisLeft(y));
|
|
||||||
//add a dot circle
|
|
||||||
svg_line.selectAll('dot')
|
|
||||||
.data(data).enter()
|
|
||||||
.append('circle')
|
|
||||||
.attr('r', 2)
|
|
||||||
.attr('cx', function(d) { return x(d.date); })
|
|
||||||
.attr('cy', function(d) { return y(d.value); })
|
|
||||||
.on('mouseover', function(d) {
|
|
||||||
div.transition().style('opacity', .9);
|
|
||||||
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
|
|
||||||
.style("left", (d3.event.pageX) + "px")
|
|
||||||
.style("top", (d3.event.pageY - 28) + "px");
|
|
||||||
})
|
|
||||||
.on('mouseout', function(d)
|
|
||||||
{
|
|
||||||
div.transition().style('opacity', 0);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
$('#date-range-to').dateRangePicker({
|
||||||
|
separator : ' to ',
|
||||||
|
getValue: function(){
|
||||||
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
|
else
|
||||||
|
return '';
|
||||||
|
},
|
||||||
|
setValue: function(s,s1,s2){
|
||||||
|
$('#date-range-from-input').val(s1);
|
||||||
|
$('#date-range-to-input').val(s2);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
{% for date in date_range %}
|
||||||
|
$('#table_{{date}}').DataTable({
|
||||||
|
"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
|
||||||
|
"iDisplayLength": 15,
|
||||||
|
"order": [[ 0, "asc" ]]
|
||||||
|
});
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
</script>
|
</script>
|
||||||
</body>
|
|
||||||
|
|
||||||
</html>
|
</html>
|
||||||
|
|
|
||||||
56
var/www/templates/crawler/show_domains_by_daterange.html
Normal file
56
var/www/templates/crawler/show_domains_by_daterange.html
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
<div class="card text-white bg-dark mb-3 mt-1">
|
||||||
|
<div class="card-header">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-6">
|
||||||
|
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type={{type}}&domains_up=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ statDomains['domains_up'] }}</a> UP
|
||||||
|
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type={{type}}&domains_down=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-danger ml-md-3">{{ statDomains['domains_down'] }}</a> DOWN
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type={{type}}&domains_up=True&domains_down=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ statDomains['total'] }}</a> Crawled
|
||||||
|
<span class="badge badge-warning ml-md-3">{{ statDomains['domains_queue'] }}</span> Queue
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title">Select domains by date range :</h5>
|
||||||
|
<p class="card-text">Some quick example text to build on the card title and make up the bulk of the card's content.</p>
|
||||||
|
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-6">
|
||||||
|
<input type="hidden" id="service_type" name="service_type" value="{{type}}">
|
||||||
|
<div class="input-group" id="date-range-from">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
<div class="input-group" id="date-range-to">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<div class="custom-control custom-switch">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="domains_up" value="True" id="domains_up_id" checked>
|
||||||
|
<label class="custom-control-label" for="domains_up_id">
|
||||||
|
<span class="badge badge-success"><i class="fas fa-check-circle"></i> Domains UP </span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="custom-control custom-switch">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="domains_down" value="True" id="domains_down_id">
|
||||||
|
<label class="custom-control-label" for="domains_down_id">
|
||||||
|
<span class="badge badge-danger"><i class="fas fa-times-circle"></i> Domains DOWN</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="custom-control custom-switch mt-2">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="domains_tags" value="True" id="domains_tags_id">
|
||||||
|
<label class="custom-control-label" for="domains_tags_id">
|
||||||
|
<span class="badge badge-dark"><i class="fas fa-tags"></i> Domains Tags</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button class="btn btn-primary">
|
||||||
|
<i class="fas fa-eye"></i> Show {{type_name}}s
|
||||||
|
</button>
|
||||||
|
<form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
Loading…
Reference in a new issue