mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [domain explorer] domains explorer v2, filter domains by daterange
This commit is contained in:
Terrtia
parent
addb885674
commit
b63fa51166
7 changed files with 525 additions and 95 deletions
|
|
@ -48,20 +48,29 @@ def get_db_keys_domain_up(domain_type, date_type): # sanitise domain_type
|
||||||
|
|
||||||
def get_list_db_keys_domain_up(domain_type, l_dates, date_type):
|
def get_list_db_keys_domain_up(domain_type, l_dates, date_type):
|
||||||
l_keys_name = []
|
l_keys_name = []
|
||||||
key_name = get_db_keys_domain_up(domain_type, date_type)
|
if domain_type=='all':
|
||||||
if key_name:
|
domains_types = get_all_domains_type()
|
||||||
for str_date in l_dates:
|
else:
|
||||||
l_keys_name.append(key_name.format(str_date))
|
domains_types = [domain_type]
|
||||||
|
|
||||||
|
for dom_type in domains_types:
|
||||||
|
key_name = get_db_keys_domain_up(dom_type, date_type)
|
||||||
|
if key_name:
|
||||||
|
for str_date in l_dates:
|
||||||
|
l_keys_name.append(key_name.format(str_date))
|
||||||
return l_keys_name
|
return l_keys_name
|
||||||
|
|
||||||
######## UTIL ########
|
######## UTIL ########
|
||||||
def sanitize_domain_type(domain_type):
|
def sanitize_domain_type(domain_type):
|
||||||
if domain_type in ['onion', 'regular']:
|
if domain_type in get_all_domains_type():
|
||||||
return domain_type
|
return domain_type
|
||||||
else:
|
else:
|
||||||
return 'regular'
|
return 'regular'
|
||||||
|
|
||||||
######## DOMAINS ########
|
######## DOMAINS ########
|
||||||
|
def get_all_domains_type():
|
||||||
|
return ['onion', 'regular']
|
||||||
|
|
||||||
def get_all_domains_up(domain_type, r_list=True):
|
def get_all_domains_up(domain_type, r_list=True):
|
||||||
'''
|
'''
|
||||||
Get all domain up (at least one time)
|
Get all domain up (at least one time)
|
||||||
|
|
@ -116,7 +125,7 @@ def get_domains_up_by_daterange(date_from, date_to, domain_type):
|
||||||
'''
|
'''
|
||||||
Get all domain up (at least one time) by daterange
|
Get all domain up (at least one time) by daterange
|
||||||
|
|
||||||
:param domain_type: date YYYYMMDD
|
:param domain_type: domain_type
|
||||||
:type domain_type: str
|
:type domain_type: str
|
||||||
|
|
||||||
:return: list of domain
|
:return: list of domain
|
||||||
|
|
@ -184,14 +193,35 @@ def domains_up_by_page(domain_type, nb_obj=28, page=1):
|
||||||
'''
|
'''
|
||||||
domains = sorted(get_all_domains_up(domain_type, r_list=False))
|
domains = sorted(get_all_domains_up(domain_type, r_list=False))
|
||||||
domains = paginate_iterator(domains, nb_obj=nb_obj, page=page)
|
domains = paginate_iterator(domains, nb_obj=nb_obj, page=page)
|
||||||
|
domains['list_elem'] = create_domains_metadata_list(domains['list_elem'], domain_type)
|
||||||
# # TODO: get tags + root_screenshot + metadata
|
|
||||||
l_domains = []
|
|
||||||
for domain in domains['list_elem']:
|
|
||||||
l_domains.append(get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=True, screenshot=True))
|
|
||||||
domains['list_elem'] = l_domains
|
|
||||||
return domains
|
return domains
|
||||||
|
|
||||||
|
def get_domains_up_by_filers(domain_type, date_from=None, date_to=None, tags=[], nb_obj=28, page=1):
|
||||||
|
if not tags:
|
||||||
|
if not date_from and not date_to:
|
||||||
|
return domains_up_by_page(domain_type, nb_obj=nb_obj, page=page)
|
||||||
|
else:
|
||||||
|
domains = sorted(get_domains_up_by_daterange(date_from, date_to, domain_type))
|
||||||
|
domains = paginate_iterator(domains, nb_obj=nb_obj, page=page)
|
||||||
|
domains['list_elem'] = create_domains_metadata_list(domains['list_elem'], domain_type)
|
||||||
|
domains['domain_type'] = domain_type
|
||||||
|
domains['date_from'] = date_from
|
||||||
|
domains['date_to'] = date_to
|
||||||
|
return domains
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def create_domains_metadata_list(list_domains, domain_type):
|
||||||
|
l_domains = []
|
||||||
|
for domain in list_domains:
|
||||||
|
if domain_type=='all':
|
||||||
|
dom_type = get_domain_type(domain)
|
||||||
|
else:
|
||||||
|
dom_type = domain_type
|
||||||
|
l_domains.append(get_domain_metadata(domain, dom_type, first_seen=True, last_ckeck=True, status=True,
|
||||||
|
ports=True, tags=True, screenshot=True, tags_safe=True))
|
||||||
|
return l_domains
|
||||||
|
|
||||||
######## DOMAIN ########
|
######## DOMAIN ########
|
||||||
|
|
||||||
def get_domain_type(domain):
|
def get_domain_type(domain):
|
||||||
|
|
@ -439,7 +469,7 @@ def get_domain_random_screenshot(domain):
|
||||||
'''
|
'''
|
||||||
return Screenshot.get_randon_domain_screenshot(domain)
|
return Screenshot.get_randon_domain_screenshot(domain)
|
||||||
|
|
||||||
def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False, screenshot=False):
|
def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False, tags_safe=False, screenshot=False):
|
||||||
'''
|
'''
|
||||||
Get Domain basic metadata
|
Get Domain basic metadata
|
||||||
|
|
||||||
|
|
@ -471,6 +501,11 @@ def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, s
|
||||||
dict_metadata['ports'] = get_domain_all_ports(domain, domain_type)
|
dict_metadata['ports'] = get_domain_all_ports(domain, domain_type)
|
||||||
if tags:
|
if tags:
|
||||||
dict_metadata['tags'] = get_domain_tags(domain)
|
dict_metadata['tags'] = get_domain_tags(domain)
|
||||||
|
if tags_safe:
|
||||||
|
if tags:
|
||||||
|
dict_metadata['is_tags_safe'] = Tag.is_tags_safe(dict_metadata['tags'])
|
||||||
|
else:
|
||||||
|
dict_metadata['is_tags_safe'] = Tag.is_tags_safe(get_domain_tags(domain))
|
||||||
if screenshot:
|
if screenshot:
|
||||||
dict_metadata['screenshot'] = get_domain_random_screenshot(domain)
|
dict_metadata['screenshot'] = get_domain_random_screenshot(domain)
|
||||||
return dict_metadata
|
return dict_metadata
|
||||||
|
|
|
||||||
|
|
@ -80,28 +80,79 @@ def showDomain():
|
||||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
||||||
modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], object_type="domain"))
|
modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], object_type="domain"))
|
||||||
|
|
||||||
@crawler_splash.route('/domains/explorer/onion', methods=['GET', 'POST'])
|
@crawler_splash.route('/domains/explorer/domain_type_post', methods=['POST'])
|
||||||
@login_required
|
@login_required
|
||||||
@login_read_only
|
@login_read_only
|
||||||
def domains_explorer_onion():
|
def domains_explorer_post_filter():
|
||||||
|
domain_onion = request.form.get('domain_onion_switch')
|
||||||
|
domain_regular = request.form.get('domain_regular_switch')
|
||||||
|
date_from = request.form.get('date_from')
|
||||||
|
date_to = request.form.get('date_to')
|
||||||
|
|
||||||
|
if date_from and date_to:
|
||||||
|
date_from = date_from.replace('-', '')
|
||||||
|
date_to = date_to.replace('-', '')
|
||||||
|
else:
|
||||||
|
date_from = None
|
||||||
|
date_to = None
|
||||||
|
|
||||||
|
if domain_onion and domain_regular:
|
||||||
|
if date_from and date_to:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_all', date_from=date_from, date_to=date_to))
|
||||||
|
else:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_all'))
|
||||||
|
elif domain_regular:
|
||||||
|
if date_from and date_to:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_web', date_from=date_from, date_to=date_to))
|
||||||
|
else:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_web'))
|
||||||
|
else:
|
||||||
|
if date_from and date_to:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_onion', date_from=date_from, date_to=date_to))
|
||||||
|
else:
|
||||||
|
return redirect(url_for('crawler_splash.domains_explorer_onion'))
|
||||||
|
|
||||||
|
@crawler_splash.route('/domains/explorer/all', methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def domains_explorer_all():
|
||||||
page = request.args.get('page')
|
page = request.args.get('page')
|
||||||
|
date_from = request.args.get('date_from')
|
||||||
|
date_to = request.args.get('date_to')
|
||||||
try:
|
try:
|
||||||
page = int(page)
|
page = int(page)
|
||||||
except:
|
except:
|
||||||
page = 1
|
page = 1
|
||||||
|
|
||||||
dict_data = Domain.domains_up_by_page('onion', page=page)
|
dict_data = Domain.get_domains_up_by_filers('all', page=page, date_from=date_from, date_to=date_to)
|
||||||
|
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='all')
|
||||||
|
|
||||||
|
@crawler_splash.route('/domains/explorer/onion', methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def domains_explorer_onion():
|
||||||
|
page = request.args.get('page')
|
||||||
|
date_from = request.args.get('date_from')
|
||||||
|
date_to = request.args.get('date_to')
|
||||||
|
try:
|
||||||
|
page = int(page)
|
||||||
|
except:
|
||||||
|
page = 1
|
||||||
|
|
||||||
|
dict_data = Domain.get_domains_up_by_filers('onion', page=page, date_from=date_from, date_to=date_to)
|
||||||
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion')
|
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion')
|
||||||
|
|
||||||
@crawler_splash.route('/domains/explorer/web', methods=['GET', 'POST'])
|
@crawler_splash.route('/domains/explorer/web', methods=['GET'])
|
||||||
@login_required
|
@login_required
|
||||||
@login_read_only
|
@login_read_only
|
||||||
def domains_explorer_web():
|
def domains_explorer_web():
|
||||||
page = request.args.get('page')
|
page = request.args.get('page')
|
||||||
|
date_from = request.args.get('date_from')
|
||||||
|
date_to = request.args.get('date_to')
|
||||||
try:
|
try:
|
||||||
page = int(page)
|
page = int(page)
|
||||||
except:
|
except:
|
||||||
page = 1
|
page = 1
|
||||||
|
|
||||||
dict_data = Domain.domains_up_by_page('regular', page=page)
|
dict_data = Domain.domains_up_by_page('regular', page=page)
|
||||||
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion')
|
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='regular')
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,13 @@
|
||||||
<!-- Core CSS -->
|
<!-- Core CSS -->
|
||||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||||
<!-- JS -->
|
<!-- JS -->
|
||||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||||
|
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
.card-columns {
|
.card-columns {
|
||||||
|
|
@ -29,84 +32,45 @@
|
||||||
|
|
||||||
<div class="col-12 col-lg-10" id="core_content">
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
<div class="card my-2 border-secondary" >
|
<div class="row">
|
||||||
<div class="card-body py-2">
|
<div class="col-12 col-lg-6">
|
||||||
<div class="row">
|
|
||||||
<div class="col-md-3 text-center">
|
|
||||||
<button class="btn btn-primary" onclick="blocks.value=0;pixelate_all();">
|
|
||||||
<i class="fas fa-eye-slash"></i>
|
|
||||||
<span class="label-icon">Hide</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
<div class="col-md-6">
|
|
||||||
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="5">
|
|
||||||
</div>
|
|
||||||
<div class="col-md-3 text-center">
|
|
||||||
<button class="btn btn-primary" onclick="blocks.value=50;pixelate_all();">
|
|
||||||
<i class="fas fa-plus-square"></i>
|
|
||||||
<span class="label-icon">Full resolution</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{% for dict_domain in dict_data['list_elem'] %}
|
{% with domain_type=dict_data['domain_type'], date_from=dict_data['date_from'], date_to=dict_data['date_to'], domain_type=domain_type%}
|
||||||
|
{% include 'domains/filter_domains.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
|
||||||
{% if loop.index0 % 4 == 0 %}
|
|
||||||
<div class="card-deck mt-3">
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
<div class="card">
|
|
||||||
<div class="text-center">
|
|
||||||
<canvas id="canvas_{{loop.index0}}" style="max-height: 400px; max-width: 100%;"></canvas>
|
|
||||||
<!-- <img style="height:400px;" src="{{url_for('showsavedpastes.screenshot', filename='a6/b9/33/f5/f1/0a/16d8b1467093dd5469bfd86bdb2c12f3694677c44406fa758f8b')}}" alt="Card image cap"> -->
|
|
||||||
</div>
|
|
||||||
<div class="card-body">
|
|
||||||
<h5 class="card-title">
|
|
||||||
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
|
|
||||||
{{dict_domain["id"]}}
|
|
||||||
</a>
|
|
||||||
</h5>
|
|
||||||
<p class="card-text">
|
|
||||||
<small class="text-muted">
|
|
||||||
First seen: {{dict_domain["first_seen"]}}<br>
|
|
||||||
Last_seen: {{dict_domain["first_seen"]}}<br>
|
|
||||||
Ports: {{dict_domain["ports"]}}
|
|
||||||
</small>
|
|
||||||
</p>
|
|
||||||
<small class="text-muted">Status: </small>
|
|
||||||
{% if dict_domain["status"] %}
|
|
||||||
<span style="color:Green;">
|
|
||||||
<i class="fas fa-check-circle"></i>
|
|
||||||
UP
|
|
||||||
</span>
|
|
||||||
{% else %}
|
|
||||||
<span style="color:Red;">
|
|
||||||
<i class="fas fa-times-circle"></i>
|
|
||||||
DOWN
|
|
||||||
</span>
|
|
||||||
{% endif %}
|
|
||||||
<div>
|
|
||||||
{% for tag in dict_domain['tags'] %}
|
|
||||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=domain<ags={{ tag }}">
|
|
||||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
|
||||||
</a>
|
|
||||||
{% endfor %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
{% if loop.index0 % 4 == 3 %}
|
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
<div class="col-12 col-xl-6">
|
||||||
|
|
||||||
{% endfor %}
|
<div class="card my-2 border-secondary" >
|
||||||
|
<div class="card-body py-2">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-3 text-center">
|
||||||
|
<button class="btn btn-primary" onclick="blocks.value=0;pixelate_all();">
|
||||||
|
<i class="fas fa-eye-slash"></i>
|
||||||
|
<span class="label-icon">Hide</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="5">
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3 text-center">
|
||||||
|
<button class="btn btn-primary" onclick="blocks.value=50;pixelate_all();">
|
||||||
|
<i class="fas fa-plus-square"></i>
|
||||||
|
<span class="label-icon">Full resolution</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{% if dict_data['list_elem']|length % 4 != 0 %}
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
|
{% with dict_data=dict_data, bootstrap_label=bootstrap_label %}
|
||||||
|
{% include 'domains/card_img_domain.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
|
||||||
<br>
|
<br>
|
||||||
<br>
|
<br>
|
||||||
|
|
@ -130,13 +94,39 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
|
$('#nav_title_domains_explorer').removeClass("text-muted");
|
||||||
|
|
||||||
|
$('#date-range-from').dateRangePicker({
|
||||||
|
separator : ' to ',
|
||||||
|
getValue: function(){
|
||||||
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
|
else
|
||||||
|
return '';
|
||||||
|
},
|
||||||
|
setValue: function(s,s1,s2){
|
||||||
|
$('#date-range-from-input').val(s1);
|
||||||
|
$('#date-range-to-input').val(s2);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
$('#date-range-to').dateRangePicker({
|
||||||
|
separator : ' to ',
|
||||||
|
getValue: function(){
|
||||||
|
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||||
|
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||||
|
else
|
||||||
|
return '';
|
||||||
|
},
|
||||||
|
setValue: function(s,s1,s2){
|
||||||
|
$('#date-range-from-input').val(s1);
|
||||||
|
$('#date-range-to-input').val(s2);
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
function toggle_sidebar(){
|
function toggle_sidebar(){
|
||||||
|
|
@ -217,15 +207,17 @@ function pixelate_img(canevas_id) {
|
||||||
function img_error(canevas_id) {
|
function img_error(canevas_id) {
|
||||||
dict_canevas_blurr_img[canevas_id]["img"].onerror=null;
|
dict_canevas_blurr_img[canevas_id]["img"].onerror=null;
|
||||||
dict_canevas_blurr_img[canevas_id]["img"].src="{{ url_for('static', filename='image/AIL.png') }}";
|
dict_canevas_blurr_img[canevas_id]["img"].src="{{ url_for('static', filename='image/AIL.png') }}";
|
||||||
dict_canevas_blurr_img[canevas_id]["img"].width = 50
|
|
||||||
}
|
}
|
||||||
|
|
||||||
blocks.addEventListener('change', pixelate_all, false);
|
blocks.addEventListener('change', pixelate_all, false);
|
||||||
|
|
||||||
|
|
||||||
{% for dict_domain in dict_data['list_elem'] %}
|
{% for dict_domain in dict_data['list_elem'] %}
|
||||||
{% if 'screenshot' in dict_domain %}
|
{% if 'screenshot' in dict_domain %}
|
||||||
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
|
{% if dict_domain['is_tags_safe'] %}
|
||||||
|
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
|
||||||
|
{% else %}
|
||||||
|
var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}";
|
||||||
|
{% endif %}
|
||||||
init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url);
|
init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url);
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
|
||||||
|
|
@ -44,5 +44,25 @@
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
<h5 class="d-flex text-muted w-100" id="nav_title_domains_explorer">
|
||||||
|
<span>Domain Explorer </span>
|
||||||
|
<a class="ml-auto" href="{{url_for('hiddenServices.manual')}}">
|
||||||
|
<i class="fas fa-plus-circle ml-auto"></i>
|
||||||
|
</a>
|
||||||
|
</h5>
|
||||||
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('crawler_splash.domains_explorer_onion')}}" id="nav_domains_explorer_onion">
|
||||||
|
<i class="fas fa-user-secret"></i>
|
||||||
|
<span>Onion Domain</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('crawler_splash.domains_explorer_web')}}" id="nav_domains_explorer_regular">
|
||||||
|
<i class="fab fa-html5"></i>
|
||||||
|
<span>Web Domain</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
52
var/www/templates/domains/card_img_domain.html
Normal file
52
var/www/templates/domains/card_img_domain.html
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
{% for dict_domain in dict_data['list_elem'] %}
|
||||||
|
|
||||||
|
{% if loop.index0 % 4 == 0 %}
|
||||||
|
<div class="card-deck mt-3">
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<div class="text-center">
|
||||||
|
<canvas id="canvas_{{loop.index0}}" style="max-height: 400px; max-width: 100%;"></canvas>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title">
|
||||||
|
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
|
||||||
|
{{dict_domain["id"]}}
|
||||||
|
</a>
|
||||||
|
</h5>
|
||||||
|
<p class="card-text">
|
||||||
|
<small class="text-muted">
|
||||||
|
First seen: {{dict_domain["first_seen"]}}<br>
|
||||||
|
Last_seen: {{dict_domain["first_seen"]}}<br>
|
||||||
|
Ports: {{dict_domain["ports"]}}
|
||||||
|
</small>
|
||||||
|
</p>
|
||||||
|
<small class="text-muted">Status: </small>
|
||||||
|
{% if dict_domain["status"] %}
|
||||||
|
<span style="color:Green;">
|
||||||
|
<i class="fas fa-check-circle"></i> UP
|
||||||
|
</span>
|
||||||
|
{% else %}
|
||||||
|
<span style="color:Red;">
|
||||||
|
<i class="fas fa-times-circle"></i> DOWN
|
||||||
|
</span>
|
||||||
|
{% endif %}
|
||||||
|
<div>
|
||||||
|
{% for tag in dict_domain['tags'] %}
|
||||||
|
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=domain<ags={{ tag }}">
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if loop.index0 % 4 == 3 %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
{% if dict_data['list_elem']|length % 4 != 0 %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
239
var/www/templates/domains/domain_explorer.html
Normal file
239
var/www/templates/domains/domain_explorer.html
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Show Domain - AIL</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.card-columns {
|
||||||
|
column-count: 4;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'crawler/menu_sidebar.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-2 border-secondary" >
|
||||||
|
<div class="card-body py-2">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-3 text-center">
|
||||||
|
<button class="btn btn-primary" onclick="blocks.value=0;pixelate_all();">
|
||||||
|
<i class="fas fa-eye-slash"></i>
|
||||||
|
<span class="label-icon">Hide</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="5">
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3 text-center">
|
||||||
|
<button class="btn btn-primary" onclick="blocks.value=50;pixelate_all();">
|
||||||
|
<i class="fas fa-plus-square"></i>
|
||||||
|
<span class="label-icon">Full resolution</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% for dict_domain in dict_data['list_elem'] %}
|
||||||
|
|
||||||
|
{% if loop.index0 % 4 == 0 %}
|
||||||
|
<div class="card-deck mt-3">
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<div class="text-center">
|
||||||
|
<canvas id="canvas_{{loop.index0}}" style="max-height: 400px; max-width: 100%;"></canvas>
|
||||||
|
<!-- <img style="height:400px;" src="{{url_for('showsavedpastes.screenshot', filename='a6/b9/33/f5/f1/0a/16d8b1467093dd5469bfd86bdb2c12f3694677c44406fa758f8b')}}" alt="Card image cap"> -->
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title">
|
||||||
|
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
|
||||||
|
{{dict_domain["id"]}}
|
||||||
|
</a>
|
||||||
|
</h5>
|
||||||
|
<p class="card-text">
|
||||||
|
<small class="text-muted">
|
||||||
|
First seen: {{dict_domain["first_seen"]}}<br>
|
||||||
|
Last_seen: {{dict_domain["first_seen"]}}<br>
|
||||||
|
Ports: {{dict_domain["ports"]}}
|
||||||
|
</small>
|
||||||
|
</p>
|
||||||
|
<small class="text-muted">Status: </small>
|
||||||
|
{% if dict_domain["status"] %}
|
||||||
|
<span style="color:Green;">
|
||||||
|
<i class="fas fa-check-circle"></i>
|
||||||
|
UP
|
||||||
|
</span>
|
||||||
|
{% else %}
|
||||||
|
<span style="color:Red;">
|
||||||
|
<i class="fas fa-times-circle"></i>
|
||||||
|
DOWN
|
||||||
|
</span>
|
||||||
|
{% endif %}
|
||||||
|
<div>
|
||||||
|
{% for tag in dict_domain['tags'] %}
|
||||||
|
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=domain<ags={{ tag }}">
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
|
</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
{% if loop.index0 % 4 == 3 %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
{% if dict_data['list_elem']|length % 4 != 0 %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<br>
|
||||||
|
<br>
|
||||||
|
|
||||||
|
{%if 'list_elem' in dict_data%}
|
||||||
|
{% with page=dict_data['page'], nb_page_max=dict_data['nb_pages'], nb_first_elem=dict_data['nb_first_elem'], nb_last_elem=dict_data['nb_last_elem'], nb_all_elem=dict_data['nb_all_elem'] %}
|
||||||
|
{% set object_name="domain" %}
|
||||||
|
{%if domain_type=='onion'%}
|
||||||
|
{% set target_url=url_for('crawler_splash.domains_explorer_onion') + "?domain_type=onion" %}
|
||||||
|
{%else%}
|
||||||
|
{% set target_url=url_for('crawler_splash.domains_explorer_web') + "?domain_type=regular" %}
|
||||||
|
{%endif%}
|
||||||
|
{% include 'pagination.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
{%endif%}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
<script>
|
||||||
|
|
||||||
|
// img_url
|
||||||
|
// ctx
|
||||||
|
// canevas_id
|
||||||
|
|
||||||
|
var dict_canevas_blurr_img = {}
|
||||||
|
|
||||||
|
function init_canevas_blurr_img(canevas_id, img_url){
|
||||||
|
|
||||||
|
// ctx, turn off image smoothin
|
||||||
|
dict_canevas_blurr_img[canevas_id] = {}
|
||||||
|
var canvas_container = document.getElementById(canevas_id);
|
||||||
|
var ctx = canvas_container.getContext('2d');
|
||||||
|
ctx.webkitImageSmoothingEnabled = false;
|
||||||
|
ctx.imageSmoothingEnabled = false;
|
||||||
|
dict_canevas_blurr_img[canevas_id]["ctx"] = ctx;
|
||||||
|
|
||||||
|
// img
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"] = new Image();
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].onload = function() {pixelate_img(canevas_id);};
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].addEventListener("error", function() {img_error(canevas_id);});
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].src = img_url;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pixelate_all(){
|
||||||
|
Object.entries(dict_canevas_blurr_img).forEach(([key, value]) => {
|
||||||
|
pixelate_img(key);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function pixelate_img(canevas_id) {
|
||||||
|
|
||||||
|
if (typeof canevas_id !== 'undefined') {
|
||||||
|
var canevas_to_blurr = document.getElementById(canevas_id);
|
||||||
|
|
||||||
|
/// use slider value
|
||||||
|
if( blocks.value == 50 ){
|
||||||
|
size = 1;
|
||||||
|
} else {
|
||||||
|
var size = (blocks.value) * 0.01;
|
||||||
|
}
|
||||||
|
|
||||||
|
canevas_to_blurr.width = dict_canevas_blurr_img[canevas_id]["img"].width;
|
||||||
|
canevas_to_blurr.height = dict_canevas_blurr_img[canevas_id]["img"].height;
|
||||||
|
|
||||||
|
/// cache scaled width and height
|
||||||
|
w = canevas_to_blurr.width * size;
|
||||||
|
h = canevas_to_blurr.height * size;
|
||||||
|
|
||||||
|
/// draw original image to the scaled size
|
||||||
|
dict_canevas_blurr_img[canevas_id]["ctx"].drawImage(dict_canevas_blurr_img[canevas_id]["img"], 0, 0, w, h);
|
||||||
|
|
||||||
|
/// pixelated
|
||||||
|
dict_canevas_blurr_img[canevas_id]["ctx"].drawImage(canevas_to_blurr, 0, 0, w, h, 0, 0, canevas_to_blurr.width, canevas_to_blurr.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
function img_error(canevas_id) {
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].onerror=null;
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].src="{{ url_for('static', filename='image/AIL.png') }}";
|
||||||
|
dict_canevas_blurr_img[canevas_id]["img"].width = 50
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks.addEventListener('change', pixelate_all, false);
|
||||||
|
|
||||||
|
|
||||||
|
{% for dict_domain in dict_data['list_elem'] %}
|
||||||
|
{% if 'screenshot' in dict_domain %}
|
||||||
|
{% if dict_domain['is_tags_safe'] %}
|
||||||
|
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
|
||||||
|
{% else %}
|
||||||
|
var screenshot_url = "{{ url_for('static', filename='image/misp-logo.png') }}";
|
||||||
|
{% endif %}
|
||||||
|
init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url);
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</html>
|
||||||
41
var/www/templates/domains/filter_domains.html
Normal file
41
var/www/templates/domains/filter_domains.html
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
<div class="card text-white bg-dark mb-3 mt-1">
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title">Filter domains by date range :</h5>
|
||||||
|
<p class="card-text">You can filter domains by a specified daterange. Display all domains if empty.</p>
|
||||||
|
<form action="{{ url_for('crawler_splash.domains_explorer_post_filter') }}" method='post'>
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-6">
|
||||||
|
|
||||||
|
<div class="input-group" id="date-range-from">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{% if date_from %}{{ date_from }}{% endif %}" name="date_from" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
<div class="input-group" id="date-range-to">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{% if date_to %}{{ date_to }}{% endif %}" name="date_to" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<div class="custom-control custom-switch">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="True" id="domain_onion_switch" {%if domain_type=='all' or domain_type=='onion'%}checked{%endif%}>
|
||||||
|
<label class="custom-control-label" for="domain_onion_switch">
|
||||||
|
<span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="custom-control custom-switch">
|
||||||
|
<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if domain_type=='all' or domain_type=='regular'%}checked{%endif%}>
|
||||||
|
<label class="custom-control-label" for="domain_regular_switch">
|
||||||
|
<span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="d-flex justify-content-between mt-2">
|
||||||
|
<button class="btn btn-outline-light" type="button" onclick="$('#date-range-from-input').val('');$('#date-range-to-input').val('');"><i class="fas fa-trash-alt"></i></button>
|
||||||
|
<button class="btn btn-secondary">
|
||||||
|
<i class="fas fa-eye"></i> Filter Domains
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
Loading…
Reference in a new issue