mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-13 01:58:22 +00:00
chg: [domain core + UI] add domain explorer v1
This commit is contained in:
Terrtia
parent
92b71e02c3
commit
addb885674
5 changed files with 370 additions and 3 deletions
|
|
@ -62,7 +62,7 @@ def sanitize_domain_type(domain_type):
|
|||
return 'regular'
|
||||
|
||||
######## DOMAINS ########
|
||||
def get_all_domains_up(domain_type):
|
||||
def get_all_domains_up(domain_type, r_list=True):
|
||||
'''
|
||||
Get all domain up (at least one time)
|
||||
|
||||
|
|
@ -72,7 +72,13 @@ def get_all_domains_up(domain_type):
|
|||
:return: list of domain
|
||||
:rtype: list
|
||||
'''
|
||||
return list(r_serv_onion.smembers("full_{}_up".format(domain_type)))
|
||||
domains = r_serv_onion.smembers("full_{}_up".format(domain_type))
|
||||
if r_list:
|
||||
if domains:
|
||||
list(domains)
|
||||
else:
|
||||
domains = []
|
||||
return domains
|
||||
|
||||
def get_domains_up_by_month(date_year_month, domain_type, rlist=False):
|
||||
'''
|
||||
|
|
@ -128,6 +134,64 @@ def get_domains_up_by_daterange(date_from, date_to, domain_type):
|
|||
domains_up = []
|
||||
return domains_up
|
||||
|
||||
def paginate_iterator(iter_elems, nb_obj=50, page=1):
|
||||
dict_page = {}
|
||||
dict_page['nb_all_elem'] = len(iter_elems)
|
||||
nb_pages = dict_page['nb_all_elem'] / nb_obj
|
||||
if not nb_pages.is_integer():
|
||||
nb_pages = int(nb_pages)+1
|
||||
else:
|
||||
nb_pages = int(nb_pages)
|
||||
if page > nb_pages:
|
||||
page = nb_pages
|
||||
|
||||
# multiple pages
|
||||
if nb_pages > 1:
|
||||
dict_page['list_elem'] = []
|
||||
start = nb_obj*(page -1)
|
||||
stop = (nb_obj*page) -1
|
||||
current_index = 0
|
||||
for elem in iter_elems:
|
||||
if current_index > stop:
|
||||
break
|
||||
if start <= current_index and stop >= current_index:
|
||||
dict_page['list_elem'].append(elem)
|
||||
current_index += 1
|
||||
stop += 1
|
||||
if stop > dict_page['nb_all_elem']:
|
||||
stop = dict_page['nb_all_elem']
|
||||
|
||||
else:
|
||||
start = 0
|
||||
stop = dict_page['nb_all_elem']
|
||||
dict_page['list_elem'] = list(iter_elems)
|
||||
dict_page['page'] = page
|
||||
dict_page['nb_pages'] = nb_pages
|
||||
# UI
|
||||
dict_page['nb_first_elem'] = start+1
|
||||
dict_page['nb_last_elem'] = stop
|
||||
return dict_page
|
||||
|
||||
def domains_up_by_page(domain_type, nb_obj=28, page=1):
|
||||
'''
|
||||
Get a list of domains up (alpha sorted)
|
||||
|
||||
:param domain_type: domain type
|
||||
:type domain_type: str
|
||||
|
||||
:return: list of domain
|
||||
:rtype: list
|
||||
'''
|
||||
domains = sorted(get_all_domains_up(domain_type, r_list=False))
|
||||
domains = paginate_iterator(domains, nb_obj=nb_obj, page=page)
|
||||
|
||||
# # TODO: get tags + root_screenshot + metadata
|
||||
l_domains = []
|
||||
for domain in domains['list_elem']:
|
||||
l_domains.append(get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=True, screenshot=True))
|
||||
domains['list_elem'] = l_domains
|
||||
return domains
|
||||
|
||||
######## DOMAIN ########
|
||||
|
||||
def get_domain_type(domain):
|
||||
|
|
@ -367,7 +431,15 @@ def get_domain_tags(domain):
|
|||
'''
|
||||
return Tag.get_obj_tag(domain)
|
||||
|
||||
def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False):
|
||||
def get_domain_random_screenshot(domain):
|
||||
'''
|
||||
Retun last screenshot (core item).
|
||||
|
||||
:param domain: crawled domain
|
||||
'''
|
||||
return Screenshot.get_randon_domain_screenshot(domain)
|
||||
|
||||
def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False, screenshot=False):
|
||||
'''
|
||||
Get Domain basic metadata
|
||||
|
||||
|
|
@ -384,6 +456,7 @@ def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, s
|
|||
:rtype: dict
|
||||
'''
|
||||
dict_metadata = {}
|
||||
dict_metadata['id'] = domain
|
||||
if first_seen:
|
||||
res = get_domain_first_seen(domain, domain_type=domain_type)
|
||||
if res is not None:
|
||||
|
|
@ -398,6 +471,8 @@ def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, s
|
|||
dict_metadata['ports'] = get_domain_all_ports(domain, domain_type)
|
||||
if tags:
|
||||
dict_metadata['tags'] = get_domain_tags(domain)
|
||||
if screenshot:
|
||||
dict_metadata['screenshot'] = get_domain_random_screenshot(domain)
|
||||
return dict_metadata
|
||||
|
||||
def get_domain_metadata_basic(domain, domain_type=None):
|
||||
|
|
|
|||
|
|
@ -76,6 +76,17 @@ def get_domain_screenshot(domain):
|
|||
else:
|
||||
return []
|
||||
|
||||
def get_randon_domain_screenshot(domain, r_path=True):
|
||||
'''
|
||||
Retun all screenshot of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
'''
|
||||
res = r_serv_onion.srandmember('domain_screenshot:{}'.format(domain))
|
||||
if res and r_path:
|
||||
return get_screenshot_rel_path(res)
|
||||
return res
|
||||
|
||||
def get_screenshot_domain(sha256_string):
|
||||
'''
|
||||
Retun all domain of a given screenshot.
|
||||
|
|
|
|||
|
|
@ -79,3 +79,29 @@ def showDomain():
|
|||
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
||||
modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], object_type="domain"))
|
||||
|
||||
@crawler_splash.route('/domains/explorer/onion', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def domains_explorer_onion():
|
||||
page = request.args.get('page')
|
||||
try:
|
||||
page = int(page)
|
||||
except:
|
||||
page = 1
|
||||
|
||||
dict_data = Domain.domains_up_by_page('onion', page=page)
|
||||
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion')
|
||||
|
||||
@crawler_splash.route('/domains/explorer/web', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def domains_explorer_web():
|
||||
page = request.args.get('page')
|
||||
try:
|
||||
page = int(page)
|
||||
except:
|
||||
page = 1
|
||||
|
||||
dict_data = Domain.domains_up_by_page('regular', page=page)
|
||||
return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion')
|
||||
|
|
|
|||
|
|
@ -124,6 +124,26 @@
|
|||
</div>
|
||||
|
||||
|
||||
<hr>
|
||||
<div class="row mb-3">
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
|
||||
<i class="fas fa-user-secret"></i> Onion Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="text-center">
|
||||
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
|
||||
<i class="fab fa-html5"></i> Web Domain Explorer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{% with object_type='domain' %}
|
||||
{% include 'tags/block_obj_tags_search.html' %}
|
||||
{% endwith %}
|
||||
|
|
|
|||
235
var/www/templates/crawler/crawler_splash/domain_explorer.html
Normal file
235
var/www/templates/crawler/crawler_splash/domain_explorer.html
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Show Domain - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.card-columns {
|
||||
column-count: 4;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'crawler/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-2 border-secondary" >
|
||||
<div class="card-body py-2">
|
||||
<div class="row">
|
||||
<div class="col-md-3 text-center">
|
||||
<button class="btn btn-primary" onclick="blocks.value=0;pixelate_all();">
|
||||
<i class="fas fa-eye-slash"></i>
|
||||
<span class="label-icon">Hide</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="5">
|
||||
</div>
|
||||
<div class="col-md-3 text-center">
|
||||
<button class="btn btn-primary" onclick="blocks.value=50;pixelate_all();">
|
||||
<i class="fas fa-plus-square"></i>
|
||||
<span class="label-icon">Full resolution</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% for dict_domain in dict_data['list_elem'] %}
|
||||
|
||||
{% if loop.index0 % 4 == 0 %}
|
||||
<div class="card-deck mt-3">
|
||||
{% endif %}
|
||||
|
||||
<div class="card">
|
||||
<div class="text-center">
|
||||
<canvas id="canvas_{{loop.index0}}" style="max-height: 400px; max-width: 100%;"></canvas>
|
||||
<!-- <img style="height:400px;" src="{{url_for('showsavedpastes.screenshot', filename='a6/b9/33/f5/f1/0a/16d8b1467093dd5469bfd86bdb2c12f3694677c44406fa758f8b')}}" alt="Card image cap"> -->
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">
|
||||
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
|
||||
{{dict_domain["id"]}}
|
||||
</a>
|
||||
</h5>
|
||||
<p class="card-text">
|
||||
<small class="text-muted">
|
||||
First seen: {{dict_domain["first_seen"]}}<br>
|
||||
Last_seen: {{dict_domain["first_seen"]}}<br>
|
||||
Ports: {{dict_domain["ports"]}}
|
||||
</small>
|
||||
</p>
|
||||
<small class="text-muted">Status: </small>
|
||||
{% if dict_domain["status"] %}
|
||||
<span style="color:Green;">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
UP
|
||||
</span>
|
||||
{% else %}
|
||||
<span style="color:Red;">
|
||||
<i class="fas fa-times-circle"></i>
|
||||
DOWN
|
||||
</span>
|
||||
{% endif %}
|
||||
<div>
|
||||
{% for tag in dict_domain['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=domain<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
{% if loop.index0 % 4 == 3 %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
|
||||
{% if dict_data['list_elem']|length % 4 != 0 %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
{%if 'list_elem' in dict_data%}
|
||||
{% with page=dict_data['page'], nb_page_max=dict_data['nb_pages'], nb_first_elem=dict_data['nb_first_elem'], nb_last_elem=dict_data['nb_last_elem'], nb_all_elem=dict_data['nb_all_elem'] %}
|
||||
{% set object_name="domain" %}
|
||||
{%if domain_type=='onion'%}
|
||||
{% set target_url=url_for('crawler_splash.domains_explorer_onion') + "?domain_type=onion" %}
|
||||
{%else%}
|
||||
{% set target_url=url_for('crawler_splash.domains_explorer_web') + "?domain_type=regular" %}
|
||||
{%endif%}
|
||||
{% include 'pagination.html' %}
|
||||
{% endwith %}
|
||||
{%endif%}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
<script>
|
||||
|
||||
// img_url
|
||||
// ctx
|
||||
// canevas_id
|
||||
|
||||
var dict_canevas_blurr_img = {}
|
||||
|
||||
function init_canevas_blurr_img(canevas_id, img_url){
|
||||
|
||||
// ctx, turn off image smoothin
|
||||
dict_canevas_blurr_img[canevas_id] = {}
|
||||
var canvas_container = document.getElementById(canevas_id);
|
||||
var ctx = canvas_container.getContext('2d');
|
||||
ctx.webkitImageSmoothingEnabled = false;
|
||||
ctx.imageSmoothingEnabled = false;
|
||||
dict_canevas_blurr_img[canevas_id]["ctx"] = ctx;
|
||||
|
||||
// img
|
||||
dict_canevas_blurr_img[canevas_id]["img"] = new Image();
|
||||
dict_canevas_blurr_img[canevas_id]["img"].onload = function() {pixelate_img(canevas_id);};
|
||||
dict_canevas_blurr_img[canevas_id]["img"].addEventListener("error", function() {img_error(canevas_id);});
|
||||
dict_canevas_blurr_img[canevas_id]["img"].src = img_url;
|
||||
}
|
||||
|
||||
function pixelate_all(){
|
||||
Object.entries(dict_canevas_blurr_img).forEach(([key, value]) => {
|
||||
pixelate_img(key);
|
||||
});
|
||||
}
|
||||
|
||||
function pixelate_img(canevas_id) {
|
||||
|
||||
if (typeof canevas_id !== 'undefined') {
|
||||
var canevas_to_blurr = document.getElementById(canevas_id);
|
||||
|
||||
/// use slider value
|
||||
if( blocks.value == 50 ){
|
||||
size = 1;
|
||||
} else {
|
||||
var size = (blocks.value) * 0.01;
|
||||
}
|
||||
|
||||
canevas_to_blurr.width = dict_canevas_blurr_img[canevas_id]["img"].width;
|
||||
canevas_to_blurr.height = dict_canevas_blurr_img[canevas_id]["img"].height;
|
||||
|
||||
/// cache scaled width and height
|
||||
w = canevas_to_blurr.width * size;
|
||||
h = canevas_to_blurr.height * size;
|
||||
|
||||
/// draw original image to the scaled size
|
||||
dict_canevas_blurr_img[canevas_id]["ctx"].drawImage(dict_canevas_blurr_img[canevas_id]["img"], 0, 0, w, h);
|
||||
|
||||
/// pixelated
|
||||
dict_canevas_blurr_img[canevas_id]["ctx"].drawImage(canevas_to_blurr, 0, 0, w, h, 0, 0, canevas_to_blurr.width, canevas_to_blurr.height);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function img_error(canevas_id) {
|
||||
dict_canevas_blurr_img[canevas_id]["img"].onerror=null;
|
||||
dict_canevas_blurr_img[canevas_id]["img"].src="{{ url_for('static', filename='image/AIL.png') }}";
|
||||
dict_canevas_blurr_img[canevas_id]["img"].width = 50
|
||||
}
|
||||
|
||||
blocks.addEventListener('change', pixelate_all, false);
|
||||
|
||||
|
||||
{% for dict_domain in dict_data['list_elem'] %}
|
||||
{% if 'screenshot' in dict_domain %}
|
||||
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}";
|
||||
init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url);
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
</script>
|
||||
|
||||
</html>
|
||||
Loading…
Reference in a new issue