chg: [Flask UI] refractor crawler frontend + show crawled domain

This commit is contained in:
Terrtia 2019-03-06 15:05:58 +01:00
parent 6f5f880b7d
commit d3b3942ad7
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
5 changed files with 269 additions and 184 deletions

View file

@ -63,7 +63,7 @@ class HiddenServices(object):
self.type = type
self.tags = {}
if type == 'onion':
if type == 'onion' or type == 'regular':
self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled"))
self.paste_crawled_directory_name = cfg.get("Directories", "crawled")
@ -75,6 +75,10 @@ class HiddenServices(object):
## TODO: # FIXME: add error
pass
def remove_absolute_path_link(self, key, value):
print(key)
print(value)
def get_origin_paste_name(self):
origin_item = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'paste_parent')
if origin_item is None:
@ -105,10 +109,35 @@ class HiddenServices(object):
for tag in p_tags:
self.tags[tag] = self.tags.get(tag, 0) + 1
def get_first_crawled(self):
res = self.r_serv_onion.zrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0, withscores=True)
if res:
res = res[0]
return {'root_item':res[0], 'epoch':res[1]}
else:
return {}
def get_last_crawled(self):
res = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0, withscores=True)
if res:
res = res[0]
return {'root_item':res[0], 'epoch':res[1]}
else:
return {}
#todo use the right paste
def get_last_crawled_pastes(self):
paste_root = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0)[0]
return self.get_all_pastes_domain(paste_root)
def get_last_crawled_pastes(self, epoch=None):
if epoch is None:
list_root = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0)
else:
list_root = self.r_serv_onion.zrevrangebyscore('crawler_history_{}:{}'.format(self.type, self.domain), int(epoch), int(epoch))
if list_root:
return self.get_all_pastes_domain(list_root[0])
else:
if epoch:
return self.get_last_crawled_pastes()
else:
return list_root
def get_all_pastes_domain(self, root_item):
if root_item is None:
@ -135,6 +164,27 @@ class HiddenServices(object):
l_crawled_pastes.extend(self.get_item_crawled_children(children))
return l_crawled_pastes
def get_item_link(self, item):
link = self.r_serv_metadata.hget('paste_metadata:{}'.format(item), 'real_link')
if link is None:
if self.paste_directory in item:
self.r_serv_metadata.hget('paste_metadata:{}'.format(item.replace(self.paste_directory+'/', '')), 'real_link')
else:
key = os.path.join(self.paste_directory, item)
link = self.r_serv_metadata.hget('paste_metadata:{}'.format(key), 'real_link')
if link:
self.remove_absolute_path_link(key, link)
return link
def get_all_links(self, l_items):
dict_links = {}
for item in l_items:
link = self.get_item_link(item)
if link:
dict_links[item] = link
return dict_links
# experimental
def get_domain_son(self, l_paste):
if l_paste is None:
@ -177,11 +227,12 @@ class HiddenServices(object):
l_screenshot_paste = []
for paste in l_crawled_pastes:
## FIXME: # TODO: remove me
origin_paste = paste
paste= paste.replace(self.paste_directory+'/', '')
paste = paste.replace(self.paste_crawled_directory_name, '')
if os.path.isfile( '{}{}.png'.format(self.screenshot_directory, paste) ):
l_screenshot_paste.append(paste[1:])
l_screenshot_paste.append({'screenshot': paste[1:], 'item': origin_paste})
if len(l_screenshot_paste) > num_screenshot:
l_random_screenshot = []

View file

@ -8,6 +8,7 @@ import redis
import datetime
import sys
import os
import time
import json
from pyfaup.faup import Faup
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
@ -95,6 +96,16 @@ def get_domain_type(domain):
else:
return 'regular'
def get_type_domain(domain):
if domain is None:
type = 'regular'
else:
if domain.rsplit('.', 1)[1] == 'onion':
type = 'onion'
else:
type = 'regular'
return type
def get_last_domains_crawled(type):
return r_serv_onion.lrange('last_{}'.format(type), 0 ,-1)
@ -560,25 +571,31 @@ def show_domains_by_daterange():
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
def onion_domain():
onion_domain = request.args.get('onion_domain')
if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
@hiddenServices.route("/hiddenServices/show_domain", methods=['GET'])
def show_domain():
domain = request.args.get('domain')
epoch = request.args.get('epoch')
type = get_type_domain(domain)
if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
return '404'
# # TODO: FIXME return 404
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check')
if last_check is None:
last_check = '********'
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
first_seen = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'first_seen')
first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen')
if first_seen is None:
first_seen = '********'
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
origin_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent')
origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
h = HiddenServices(onion_domain, 'onion')
l_pastes = h.get_last_crawled_pastes()
h = HiddenServices(domain, type)
last_crawled_time = h.get_last_crawled()
if epoch in last_crawled_time:
last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(last_crawled_time['epoch'])))
l_pastes = h.get_last_crawled_pastes(epoch=epoch)
dict_links = h.get_all_links(l_pastes)
if l_pastes:
status = True
else:
@ -600,8 +617,9 @@ def onion_domain():
p_tags = r_serv_metadata.smembers('tag:'+path)
paste_tags.append(unpack_paste_tags(p_tags))
return render_template("showDomain.html", domain=onion_domain, last_check=last_check, first_seen=first_seen,
return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
dict_links=dict_links,
path_name=path_name, origin_paste_tags=origin_paste_tags, status=status,
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
domain_tags=domain_tags, screenshot=screenshot)

View file

@ -62,7 +62,7 @@
<tbody id="tbody_last_crawled">
{% for metadata_domain in last_domains %}
<tr data-toggle="popover" data-trigger="hover" title="<span class='badge badge-dark'>{{metadata_domain['domain']}}</span>" data-content="{{metadata_domain['epoch']}}">
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_domain['domain'] }}">{{ metadata_domain['domain_name'] }}</a></td>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@ -255,7 +255,7 @@ function refresh_list_crawled(){
var newRow = tableRef.insertRow(tableRef.rows.length);
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
@ -293,7 +293,7 @@ function refresh_list_crawled(){
newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";

View file

@ -1,60 +1,53 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Show Domain - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js')}}"></script>
<style>
.test thead{
background: #d91f2d;
color: #fff;
}
</style>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'navbar.html' %}
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div id="page-wrapper">
<div class="row">
<div class="col-md-6">
<div class="row">
<div class="panel panel-info">
<div class="panel-heading">
<div class="col-12 col-xl-6">
<div class="card mt-2">
<div class="card-header bg-dark">
<span class="badge badge-pill badge-light flex-row-reverse float-right">
{% if status %}
<div class="pull-right" style="color:Green;">
<i class="fa fa-check-circle fa-2x"></i>
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
UP
</div>
{% else %}
<div class="pull-right" style="color:Red;">
<i class="fa fa-times-circle fa-2x"></i>
<div style="color:Red;">
<i class="fas fa-times-circle fa-2x"></i>
DOWN
</div>
{% endif %}
<h3>{{ domain }} :</h3>
<ul class="list-group">
<li class="list-group-item">
<table class="table table-condensed">
</span>
<h3 class="card-title text-white">{{ domain }} :</h3>
</div>
<div class="card-body">
<table class="table table-responsive table-condensed">
<thead>
<tr>
<th>First Seen</th>
@ -63,76 +56,80 @@
</thead>
<tbody>
<tr>
<td class="panelText"><a href="#">{{ first_seen }}</a></td>
<td class="panelText"><a href="#">{{ last_check }}</a></td>
<td class="panelText">{{ first_seen }}</td>
<td class="panelText">{{ last_check }}</td>
</tr>
</tbody>
</table>
</li>
<li class="list-group-item">
Origin Paste: <a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
Origin Paste:
{% if origin_paste_name=='manual' or origin_paste_name=='auto' %}
<span class="badge badge-dark">{{ origin_paste_name }}</span>
{%else%}
<a class="badge badge-dark" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
{%endif%}
<div>
{% for tag in origin_paste_tags %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
<br>
</div>
</li>
</ul>
</div>
</div>
<div>
{% for tag in domain_tags %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
</a>
{% endfor %}
<br>
<br>
</div>
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_">
<thead>
{% if l_pastes %}
<table class="table table-striped table-bordered table-hover" id="myTable_1">
<thead class="thead-dark">
<tr>
<th style="max-width: 800px;">Crawled Pastes</th>
<th>Crawled Pastes</th>
</tr>
</thead>
<tbody>
{% for path in l_pastes %}
<tr>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path_name[loop.index0] }}</a>
<td>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
<div style="line-height:0.9;">{{ dict_links[path] }}</div>
</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{%endif%}
</div>
</div>
<div class="col-md-6">
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
<div class="col-12 col-xl-6">
<div class="card my-2" style="background-color:#ecf0f1;">
<div class="card-body py-2">
<div class="row">
<div class="col-md-8">
<input class="center" id="blocks" type="range" min="1" max="50" value="13">
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="13">
</div>
<div class="col-md-4">
<button class="btn btn-primary btn-tags" onclick="blocks.value=50;pixelate();">
<span class="glyphicon glyphicon-zoom-in"></span>
<button class="btn btn-primary" onclick="blocks.value=50;pixelate();">
<i class="fas fa-search-plu"></i>
<span class="label-icon">Full resolution</span>
</button>
</div>
@ -140,25 +137,47 @@
</div>
</div>
<canvas id="canvas" style="width:100%;"></canvas>
<div class="text-center">
<small>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
</a>
<small>
</div>
</div>
</div>
</div>
<!-- /#page-wrapper -->
</div>
</div>
</body>
<script>
var table;
$(document).ready(function(){
activePage = "page-hiddenServices"
$("#"+activePage).addClass("active");
table = $('#myTable_').DataTable(
table = $('#myTable_1').DataTable(
{
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
"iDisplayLength": 5,
"order": [[ 0, "desc" ]]
}
);
//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
//"iDisplayLength": 5,
//"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
@ -172,10 +191,9 @@
img.addEventListener("error", img_error);
var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot) }}";
img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
function pixelate() {
/// use slider value
if( blocks.value == 50 ){
size = 1;
@ -208,6 +226,4 @@
}
</script>
</body>
</html>

View file

@ -433,7 +433,7 @@
<tbody>
<tr>
<td>Domain</td>
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ crawler_metadata['domain'] }}" id='onion_domain'>{{ crawler_metadata['domain'] }}</a></td>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
</tr>
<tr>
<td>Father</td>