mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [hiddenServices] auto update last crawled domains
This commit is contained in:
parent
b3b75ccbea
commit
46c1105483
2 changed files with 127 additions and 22 deletions
|
@ -102,7 +102,40 @@ def hiddenServices_page():
|
||||||
metadata_onion['status_icon'] = 'fa-times-circle'
|
metadata_onion['status_icon'] = 'fa-times-circle'
|
||||||
list_onion.append(metadata_onion)
|
list_onion.append(metadata_onion)
|
||||||
|
|
||||||
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains)
|
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
||||||
|
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains, date_from=date_string, date_to=date_string)
|
||||||
|
|
||||||
|
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_jon", methods=['GET'])
|
||||||
|
def last_crawled_domains_with_stats_jon():
|
||||||
|
last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
|
||||||
|
list_onion = []
|
||||||
|
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
|
||||||
|
statDomains = {}
|
||||||
|
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
|
||||||
|
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
|
||||||
|
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
||||||
|
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
|
||||||
|
|
||||||
|
for onion in last_onions:
|
||||||
|
metadata_onion = {}
|
||||||
|
metadata_onion['domain'] = onion
|
||||||
|
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
|
||||||
|
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
|
||||||
|
if get_onion_status(onion, metadata_onion['last_check']):
|
||||||
|
metadata_onion['status_text'] = 'UP'
|
||||||
|
metadata_onion['status_color'] = 'Green'
|
||||||
|
metadata_onion['status_icon'] = 'fa-check-circle'
|
||||||
|
else:
|
||||||
|
metadata_onion['status_text'] = 'DOWN'
|
||||||
|
metadata_onion['status_color'] = 'Red'
|
||||||
|
metadata_onion['status_icon'] = 'fa-times-circle'
|
||||||
|
list_onion.append(metadata_onion)
|
||||||
|
|
||||||
|
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
||||||
|
|
||||||
|
return jsonify({'last_onions': list_onion, 'statDomains': statDomains})
|
||||||
|
|
||||||
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
|
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
|
||||||
def get_onions_by_daterange():
|
def get_onions_by_daterange():
|
||||||
|
|
|
@ -66,7 +66,7 @@
|
||||||
<th>Status</th>
|
<th>Status</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody id="tbody_last_crawled">
|
||||||
{% for metadata_onion in last_onions %}
|
{% for metadata_onion in last_onions %}
|
||||||
<tr>
|
<tr>
|
||||||
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_onion['domain'] }}">{{ metadata_onion['domain'] }}</a></td>
|
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_onion['domain'] }}">{{ metadata_onion['domain'] }}</a></td>
|
||||||
|
@ -152,37 +152,53 @@
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
|
||||||
<div style="color:Green; display:inline-block">
|
<div style="color:Green; display:inline-block">
|
||||||
<i class="fa fa-check-circle fa-2x"></i>
|
<i class="fa fa-check-circle fa-2x"></i>
|
||||||
Domains UP
|
Domains UP
|
||||||
</div>
|
</div>
|
||||||
|
</a>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
|
||||||
<div style="color:Green; display:inline-block">
|
<div style="color:Green; display:inline-block">
|
||||||
{{ statDomains['domains_up'] }}
|
<div id="text_domain_up">{{ statDomains['domains_up'] }}</div>
|
||||||
</div>
|
</div>
|
||||||
|
</a>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
|
||||||
<div style="color:Red; display:inline-block">
|
<div style="color:Red; display:inline-block">
|
||||||
<i class="fa fa-times-circle fa-2x"></i>
|
<i class="fa fa-times-circle fa-2x"></i>
|
||||||
Domains DOWN
|
Domains DOWN
|
||||||
</div>
|
</div>
|
||||||
|
</a>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
|
||||||
<div style="color:Red; display:inline-block">
|
<div style="color:Red; display:inline-block">
|
||||||
{{ statDomains['domains_down'] }}
|
<div id="text_domain_down">{{ statDomains['domains_down'] }}</div>
|
||||||
</div>
|
</div>
|
||||||
|
</a>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Crawled Domains</td>
|
<td>
|
||||||
<td>{{ statDomains['total'] }}</td>
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
|
||||||
|
Crawled Domains
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
|
||||||
|
<div id="text_total_domains">{{ statDomains['total'] }}</div>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Domains in Queue</td>
|
<td>Domains in Queue</td>
|
||||||
<td>{{ statDomains['domains_queue'] }}</td>
|
<td><div id="text_domain_queue">{{ statDomains['domains_queue'] }}</div></td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
@ -196,6 +212,7 @@
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
var all_graph = {};
|
var all_graph = {};
|
||||||
|
var to_refresh = false
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
activePage = "page-hiddenServices"
|
activePage = "page-hiddenServices"
|
||||||
$("#"+activePage).addClass("active");
|
$("#"+activePage).addClass("active");
|
||||||
|
@ -232,10 +249,22 @@
|
||||||
});
|
});
|
||||||
|
|
||||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
||||||
|
|
||||||
|
$( window ).focus(function() {
|
||||||
|
to_refresh = true
|
||||||
|
refresh_list_crawled();
|
||||||
|
});
|
||||||
|
$( window ).blur(function() {
|
||||||
|
to_refresh = false
|
||||||
|
});
|
||||||
|
|
||||||
|
to_refresh = true
|
||||||
|
refresh_list_crawled();
|
||||||
});
|
});
|
||||||
$(window).on("resize", function() {
|
$(window).on("resize", function() {
|
||||||
all_graph.onResize();
|
all_graph.onResize();
|
||||||
});
|
});
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
|
@ -319,6 +348,49 @@ d3.json(url)
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
|
||||||
|
function refresh_list_crawled(){
|
||||||
|
|
||||||
|
$.getJSON("{{ url_for('hiddenServices.last_crawled_domains_with_stats_jon') }}",
|
||||||
|
function(data) {
|
||||||
|
|
||||||
|
var tableRef = document.getElementById('tbody_last_crawled');
|
||||||
|
$("#tbody_last_crawled").empty()
|
||||||
|
|
||||||
|
for (var i = 0; i < data.last_onions.length; i++) {
|
||||||
|
var data_domain = data.last_onions[i]
|
||||||
|
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||||
|
|
||||||
|
var newCell = newRow.insertCell(0);
|
||||||
|
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
|
||||||
|
|
||||||
|
newCell = newRow.insertCell(1);
|
||||||
|
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
|
||||||
|
|
||||||
|
newCell = newRow.insertCell(2);
|
||||||
|
newCell.innerHTML = "<td>"+data_domain['last_check'].substr(0, 4)+"/"+data_domain['last_check'].substr(4, 2)+"/"+data_domain['last_check'].substr(6, 2)+"</td>"
|
||||||
|
|
||||||
|
newCell = newRow.insertCell(3);
|
||||||
|
newCell.innerHTML = "<td><div style=\"color:"+data_domain['status_color']+"; display:inline-block\"><i class=\"fa "+data_domain['status_icon']+" fa-2x\"></i>"+data_domain['status_text']+"</div></td>"
|
||||||
|
|
||||||
|
}
|
||||||
|
var statDomains = data.statDomains
|
||||||
|
document.getElementById('text_domain_up').innerHTML = statDomains['domains_up']
|
||||||
|
document.getElementById('text_domain_down').innerHTML = statDomains['domains_down']
|
||||||
|
document.getElementById('text_domain_queue').innerHTML = statDomains['domains_queue']
|
||||||
|
document.getElementById('text_total_domains').innerHTML = statDomains['total']
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (to_refresh) {
|
||||||
|
setTimeout("refresh_list_crawled()", 10000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue