AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into user_management

Browse source
This commit is contained in:
Terrtia 2019-06-19 15:11:44 +02:00
commit 6eec0aabfe
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
9 changed files with 496 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
bin/MISP_The_Hive_feeder.py
View file

@ -55,21 +55,22 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
def create_the_hive_alert(source, path, tag):
tags = list(r_serv_metadata.smembers('tag:'+path))
# # TODO: check items status (processed by all modules)
# # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation...
# # # TODO: description, add AIL link:show items ?
tags = list( r_serv_metadata.smembers('tag:{}'.format(path)) )
artifacts = [
AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
AlertArtifact( dataType='file', data=path, tags=tags )
]
l_tags = tag.split(',')
# Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title='AIL Leak',
tlp=3,
tags=l_tags,
description='infoleak',
tags=tags,
description='AIL Leak, triggered by {}'.format(tag),
type='ail',
source=source,
sourceRef=sourceRef,

7
bin/PgpDump.py
View file

@ -82,7 +82,12 @@ def get_pgp_packet(message, save_path):
process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE)
process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE)
process1.stdout.close()
output = process2.communicate()[0].decode()
output = process2.communicate()[0]
try:
output = output.decode()
except UnicodeDecodeError:
publisher.error('Error PgpDump UnicodeDecodeError: {}'.format(message))
output = ''
return output
def get_pgp_packet_file(file):

121
bin/packages/HiddenServices.py
View file

@ -17,10 +17,14 @@ Conditions to fulfill to be able to use this class correctly:
"""
import os
import time
import gzip
import redis
import random
from io import BytesIO
import zipfile
import configparser
import sys
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
@ -71,6 +75,7 @@ class HiddenServices(object):
self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled"))
self.paste_crawled_directory_name = cfg.get("Directories", "crawled")
self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
self.screenshot_directory_screenshot = os.path.join(self.screenshot_directory, 'screenshot')
elif type == 'i2p':
self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
@ -120,6 +125,26 @@ class HiddenServices(object):
for tag in p_tags:
self.tags[tag] = self.tags.get(tag, 0) + 1
def extract_epoch_from_history(self, crawled_history):
epoch_list = []
for res, epoch_val in crawled_history:
epoch_val = int(epoch_val) # force int
try:
# domain down
if int(res) == epoch_val:
status = False
# domain up
else:
status = True
except ValueError:
status = True
epoch_val = int(epoch_val) # force int
epoch_list.append((epoch_val, time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), status))
return epoch_list
def get_domain_crawled_history(self):
return self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, -1, withscores=True)
def get_first_crawled(self):
res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
if res:
@ -230,6 +255,13 @@ class HiddenServices(object):
return l_crawled_pastes
'''
def get_item_screenshot(self, item):
screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot')
if screenshot:
screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
return screenshot
return ''
def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1):
l_screenshot_paste = []
for paste in l_crawled_pastes:
@ -237,9 +269,8 @@ class HiddenServices(object):
origin_paste = paste
paste= paste.replace(self.paste_directory+'/', '')
screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(paste), 'screenshot')
screenshot = self.get_item_screenshot(paste)
if screenshot:
screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
l_screenshot_paste.append({'screenshot': screenshot, 'item': origin_paste})
if len(l_screenshot_paste) > num_screenshot:
@ -250,6 +281,35 @@ class HiddenServices(object):
else:
return l_screenshot_paste
def get_all_domain_screenshot(self, l_crawled_pastes, filename=False):
l_screenshot_paste = []
for paste in l_crawled_pastes:
## FIXME: # TODO: remove me
origin_paste = paste
paste= paste.replace(self.paste_directory+'/', '')
screenshot = self.get_item_screenshot(paste)
if screenshot:
screenshot = screenshot + '.png'
screenshot_full_path = os.path.join(self.screenshot_directory_screenshot, screenshot)
if filename:
screen_file_name = os.path.basename(paste) + '.png'
l_screenshot_paste.append( (screenshot_full_path, screen_file_name) )
else:
l_screenshot_paste.append(screenshot_full_path)
return l_screenshot_paste
def get_all_item_full_path(self, l_items, filename=False):
l_full_items = []
for item in l_items:
item = os.path.join(self.PASTES_FOLDER, item)
if filename:
file_name = os.path.basename(item) + '.gz'
l_full_items.append( (item, file_name) )
else:
l_full_items.append(item)
return l_full_items
def get_crawled_pastes_by_date(self, date):
pastes_path = os.path.join(self.paste_crawled_directory, date[0:4], date[4:6], date[6:8])
@ -258,6 +318,63 @@ class HiddenServices(object):
l_crawled_pastes = []
return l_crawled_pastes
def get_all_har(self, l_pastes, filename=False):
all_har = []
for item in l_pastes:
if filename:
all_har.append( (self.get_item_har(item), os.path.basename(item) + '.json') )
else:
all_har.append(self.get_item_har(item))
return all_har
def get_item_har(self, item_path):
item_path = item_path.replace('{}/'.format(self.paste_crawled_directory_name), '', 1)
har_path = os.path.join(self.screenshot_directory, item_path) + '.json'
return har_path
def create_domain_basic_archive(self, l_pastes):
all_har = self.get_all_har(l_pastes, filename=True)
all_screenshot = self.get_all_domain_screenshot(l_pastes, filename=True)
all_items = self.get_all_item_full_path(l_pastes, filename=True)
# try:
# zip buffer
zip_buffer = BytesIO()
with zipfile.ZipFile(zip_buffer, "a") as zf:
#print(all_har)
self.write_in_zip_buffer(zf, all_har)
self.write_in_zip_buffer(zf, all_screenshot)
self.write_in_zip_buffer(zf, all_items)
# write map url
map_file_content = self.get_metadata_file(l_pastes).encode()
zf.writestr( '_URL_MAP_', BytesIO(map_file_content).getvalue())
zip_buffer.seek(0)
return zip_buffer
# except Exception as e:
# print(e)
# return 'Server Error'
def write_in_zip_buffer(self, zf, list_file):
for file_path, file_name in list_file:
with open(file_path, "rb") as f:
har_content = f.read()
zf.writestr( file_name, BytesIO(har_content).getvalue())
def get_metadata_file(self, list_items):
file_content = ''
dict_url = self.get_all_links(list_items)
for key in dict_url:
file_content = '{}\n{} : {}'.format(file_content, os.path.basename(key), dict_url[key])
return file_content
'''
def get_last_crawled_pastes_fileSearch(self):

320
var/www/modules/dashboard/templates/index.html
View file

@ -2,23 +2,40 @@
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
data-urlstuff="{{ url_for('dashboard.stuff') }}" data-urllog="{{ url_for('dashboard.logs') }}">
</script>
<style>
.tableQueue tbody tr td,
.tableQueue tbody tr th,
.tableQueue thead tr td,
.tableQueue thead tr th{
padding: 1px;
}
tr.table-disabled {
background-color: #f2f2f2;
}
tr.table-log-warning {
background-color: #ccfae3;
}
</style>
<script>
window.default_minute = {{ default_minute }};
window.glob_tabvar = []; // Avoid undefined
@ -31,184 +48,127 @@
};
update_values();
</script>
<style>
.tableQueue tbody tr td,
.tableQueue tbody tr th,
.tableQueue thead tr td,
.tableQueue thead tr th{
padding: 1px;
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-inverse navbar-static-top nav">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
{% include 'searchbox.html' %}
<!-- /input-group -->
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-dashboard fa-fw"></i> Total pastes since {{ default_minute }} min
</div>
<div class="panel-body">
<div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
</div>
</div>
<!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
<div class='pull_right'>
<label class="switch">
<input id="button-toggle-queues" class="switch-input" type="checkbox" checked>
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">Display queues</strong>
<div>
<div>
<table id="queue-color-legend" class="table">
<thead>
</thead>
<tbody>
<tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #faf2cc;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
</tbody>
</table>
</div>
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
</br>
</div>
</div>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'dashboard/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{%if update_in_progress%}
<div class="alert alert-warning alert-dismissible fade in">
<a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
<div class="alert alert-warning alert-dismissible fade show my-2" role="alert">
<strong>Warning!</strong> {{update_warning_message}} <strong>{{update_warning_message_notice_me}}</strong>
(<a href="{{ url_for('settings.settings_page') }}">Check Update Status</a>)
<button type="button" class="close" data-dismiss="alert" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
{%endif%}
<div class="alert alert-info alert-dismissible fade in">
<a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
<strong>Bootstrap 4 migration!</strong> Some pages are still in bootstrap 3. You can check the migration progress <strong><a href="https://github.com/CIRCL/AIL-framework/issues/330" target="_blank">Here</a></strong>.
<div class="alert alert-info alert-dismissible fade show mt-1" role="alert">
<strong>Bootstrap 4 migration!</strong> Some pages are still in bootstrap 3. You can check the migration progress <strong><a href="https://github.com/CIRCL/AIL-framework/issues/330" target="_blank">Here</a></strong>.
<button type="button" class="close" data-dismiss="alert" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
<div class="row">
<div class="col-lg-6">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Feeder(s) Monitor:
</div>
<div id="panelbody" class="panel-body" style="height:420px;">
<strong>Processed pastes</strong>
<div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
<hr style="border-top: 2px solid #eee; margin-top: 7px; margin-bottom: 7px;">
<strong>Filtered duplicates</strong>
<div id="Dup_feeder" style="height: 100px; padding: 0px; position: relative;"></div>
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
</div>
<!-- /.col-lg-6 -->
<div class="col-lg-6">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Queues Monitor
</div>
<div class="panel-body">
<div class="row">
<div class="col-lg-6" id="Graph" style="height:195px; width:88%;"></div>
<div style="height:10px;"></div>
<div class="col-lg-6" id="Graph2" style="height:195px; width:88%;"></div>
</div>
<!-- /.row -->
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
</div>
<!-- /.col-lg-6 -->
<div class="col-lg-12">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-tasks fa-fw"></i> Logs
<div class="pull-right">
<label style="padding-bottom:2px;">
<select class="form-control input-sm" id="log_select">
{% for log_selection in log_select %}
{% if log_selection == selected %}
<option value="{{ log_selection }}" selected>{{ log_selection }}</option>
{% else %}
<option value="{{ log_selection }}">{{ log_selection }}</option>
{% endif %}
{% endfor %}
</select>
</label>
<input id="checkbox_log_info" type="checkbox" value="info"> INFO
<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL
<div class="row my-2">
<div class="col-xl-6">
<div class="card">
<div class="card-header">
<i class="far fa-chart-bar"></i> Feeder(s) Monitor:
</div>
<div class="card-body" id="panelbody" style="height:420px;">
<strong>Processed pastes</strong>
<div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
<hr style="border-top: 2px solid #eee; margin-top: 7px; margin-bottom: 7px;">
<strong>Filtered duplicates</strong>
<div id="Dup_feeder" style="height: 100px; padding: 0px; position: relative;"></div>
</div>
</div>
</div>
<div class="col-xl-6">
<div class="card">
<div class="card-header">
<i class="far fa-chart-bar"></i> Queues Monitor
</div>
<div class="card-body">
<div class="" id="Graph" style="height:195px; width:88%;"></div>
<div style="height:10px;"></div>
<div class="" id="Graph2" style="height:195px; width:88%;"></div>
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-header">
<i class="fas fa-clipboard-list"></i> Logs
<div class="float-right">
<label style="padding-bottom:2px;">
<select class="form-control input-sm" id="log_select">
{% for log_selection in log_select %}
{% if log_selection == selected %}
<option value="{{ log_selection }}" selected>{{ log_selection }}</option>
{% else %}
<option value="{{ log_selection }}">{{ log_selection }}</option>
{% endif %}
{% endfor %}
</select>
</label>
<input id="checkbox_log_info" type="checkbox" value="INFO"> INFO
<input id="checkbox_log_warning" type="checkbox" value="WARNING" checked="true"> WARNING
<input id="checkbox_log_critical" type="checkbox" value="CRITICAL" checked="true"> CRITICAL
</div>
</div>
<div class="card-body">
<table class="table table-hover table-sm" id="table_log">
<thead>
<tr>
<th>Time</th>
<th>Channel</th>
<th>Level</th>
<th>Script Name</th>
<th>Source</th>
<th>Date</th>
<th>Paste name</th>
<th>Message</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="tab_body">
</tbody>
</table>
</div>
</div>
</div>
<div class="panel-body">
<div class="row">
<div class="col-lg-12">
<table class="table table-bordered table-hover table-striped" id="table_log">
<thead>
<tr>
<th>Time</th>
<th>Channel</th>
<th>Level</th>
<th>Script Name</th>
<th>Source</th>
<th>Date</th>
<th>Paste name</th>
<th>Message</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="tab_body">
</tbody>
</table>
</div>
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
</div>
<script> var url_showSavedPath = "{{ url_for('showsavedpastes.showsavedpaste') }}"; </script>
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
data-urlstuff="{{ url_for('dashboard.stuff') }}" data-urllog="{{ url_for('dashboard.logs') }}">
</script>
</body>
<script> var url_showSavedPath = "{{ url_for('showsavedpastes.showsavedpaste') }}"; </script>
<script>
activePage = "page-index";
$("#"+activePage).addClass("active");
$("#page-Dashboard").addClass("active");
var tableBody = document.getElementById('tab_body')
$.getJSON("{{ url_for('dashboard.get_last_logs_json') }}", function(data) {
@ -224,7 +184,7 @@
var msage = document.createElement('TD')
var inspect = document.createElement('TD')
tr.className = "warning";
tr.className = "table-log-warning";
time.appendChild(document.createTextNode(d.time))
chan.appendChild(document.createTextNode('Script'))
level.appendChild(document.createTextNode('WARNING'))
@ -236,10 +196,10 @@
var iconspan = document.createElement('SPAN');
var message = d.message.split(" ")
if (message[0] == "Detected"){
iconspan.className = "glyphicon glyphicon-eye-open";
iconspan.className = "fas fa-eye";
}
else if (message[0] == "Checked"){
iconspan.className = "glyphicon glyphicon-thumbs-up";
iconspan.className = "far fa-thumbs-up";
}
iconspan.innerHTML = "&nbsp;";
msage.appendChild(iconspan);
@ -249,7 +209,7 @@
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", d.path);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fa fa-search-plus";
action_icon_span.className = "fas fa-search-plus";
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a)
inspect.setAttribute("style", "text-align:center;");
@ -269,14 +229,22 @@
});
});
</script>
<script type="text/javascript">
initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
</script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side-bard-dashboard-content').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side-bard-dashboard-content').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

59
var/www/modules/hiddenServices/Flask_hiddenServices.py
View file

@ -11,7 +11,7 @@ import os
import time
import json
from pyfaup.faup import Faup
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for
from flask_login import login_required
from Date import Date
@ -764,6 +764,7 @@ def show_domain():
if first_seen is None:
first_seen = '********'
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
h = HiddenServices(domain, type, port=port)
@ -793,13 +794,67 @@ def show_domain():
p_tags = r_serv_metadata.smembers('tag:'+path)
paste_tags.append(unpack_paste_tags(p_tags))
domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
dict_links=dict_links,
dict_links=dict_links, port=port, epoch=epoch,
ports=ports, domain_history=domain_history,
origin_paste_tags=origin_paste_tags, status=status,
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
domain_tags=domain_tags, screenshot=screenshot)
@hiddenServices.route("/crawlers/download_domain", methods=['GET'])
def download_domain():
domain = request.args.get('domain')
epoch = request.args.get('epoch')
try:
epoch = int(epoch)
except:
epoch = None
port = request.args.get('port')
faup.decode(domain)
unpack_url = faup.get()
## TODO: # FIXME: remove me
try:
domain = unpack_url['domain'].decode()
except:
domain = unpack_url['domain']
if not port:
if unpack_url['port']:
try:
port = unpack_url['port'].decode()
except:
port = unpack_url['port']
else:
port = 80
try:
port = int(port)
except:
port = 80
type = get_type_domain(domain)
if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
return '404'
# # TODO: FIXME return 404
origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
h = HiddenServices(domain, type, port=port)
item_core = h.get_domain_crawled_core_item(epoch=epoch)
if item_core:
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
else:
l_pastes = []
#dict_links = h.get_all_links(l_pastes)
zip_file = h.create_domain_basic_archive(l_pastes)
filename = domain + '.zip'
return send_file(zip_file, attachment_filename=filename, as_attachment=True)
@hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
@login_required
def onion_son():

38
var/www/modules/hiddenServices/templates/showDomain.html
View file

@ -53,12 +53,14 @@
<tr>
<th>First Seen</th>
<th>Last Check</th>
<th>Ports</th>
</tr>
</thead>
<tbody>
<tr>
<td class="panelText">{{ first_seen }}</td>
<td class="panelText">{{ last_check }}</td>
<td class="panelText">{{ ports }}</td>
</tr>
</tbody>
</table>
@ -119,6 +121,42 @@
</table>
{%endif%}
<div class="text-center my-3">
<a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
</button>
</a>
</div>
{% if domain_history %}
<table class="table table-hover mt-2" id="myTable_2">
<thead>
<tr>
<th><i class="fas fa-history"></i> Domain History</th>
</tr>
</thead>
<tbody>
{% for epoch_item in domain_history %}
<tr>
<td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
<div class="d-flex justify-content-around" style="line-height:0.9;">
<div>{{domain}}</div>
{% if epoch_item[2] %}
<div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
{% else %}
<div style="color:Red;"><i class="fas fa-times-circle"></i> DOWN</div>
{% endif %}
<div>{{ epoch_item[1] }}</div>
</div>
</a>
</tr>
{% endfor %}
</tbody>
</table>
{%endif%}
</div>
<div class="col-12 col-xl-6">

177
var/www/static/js/indexjavascript.js
View file

@ -201,92 +201,94 @@ function create_log_table(obj_json) {
return;
}
if( chansplit[1] == "INFO" ){
tr.className = "info";
}
else if ( chansplit[1] == "WARNING" ){
tr.className = "warning";
}
else if ( chansplit[1] == "CRITICAL"){
tr.className = "danger"
}
if( parsedmess.length>2 ){
if( chansplit[1] == "INFO" ){
tr.className = "table-disabled";
}
else if ( chansplit[1] == "WARNING" ){
tr.className = "table-log-warning";
}
else if ( chansplit[1] == "CRITICAL"){
tr.className = "table-danger"
}
source_link = document.createElement("A");
if (parsedmess[1] == "slexy.org"){
source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0];
}
else{
source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
}
source_link.setAttribute("HREF",source_url);
source_link.setAttribute("TARGET", "_blank");
source_link.appendChild(document.createTextNode(parsedmess[1]));
source_link = document.createElement("A");
if (parsedmess[1] == "slexy.org"){
source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0];
}
else{
source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
}
source_link.setAttribute("HREF",source_url);
source_link.setAttribute("TARGET", "_blank");
source_link.appendChild(document.createTextNode(parsedmess[1]));
src.appendChild(source_link);
src.appendChild(source_link);
var now = new Date();
var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds());
var now = new Date();
var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds());
time.appendChild(document.createTextNode(timepaste));
chan.appendChild(document.createTextNode(chansplit[0]));
level.appendChild(document.createTextNode(chansplit[1]));
time.appendChild(document.createTextNode(timepaste));
chan.appendChild(document.createTextNode(chansplit[0]));
level.appendChild(document.createTextNode(chansplit[1]));
scrpt.appendChild(document.createTextNode(parsedmess[0]));
pdate.appendChild(document.createTextNode(parsedmess[2]));
nam.appendChild(document.createTextNode(parsedmess[3]));
scrpt.appendChild(document.createTextNode(parsedmess[0]));
pdate.appendChild(document.createTextNode(parsedmess[2]));
nam.appendChild(document.createTextNode(parsedmess[3]));
var iconspan = document.createElement('SPAN');
if (parsedmess[4].split(" ")[0] == "Detected"){
iconspan.className = "glyphicon glyphicon-eye-open";
}
else if (parsedmess[4].split(" ")[0] == "Checked"){
iconspan.className = "glyphicon glyphicon-thumbs-up";
}
iconspan.innerHTML = "&nbsp;";
msage.appendChild(iconspan);
var message = parsedmess[4].split(" ");
message.shift();
var iconspan = document.createElement('SPAN');
if (parsedmess[4].split(" ")[0] == "Detected"){
iconspan.className = "fas fa-eye";
}
else if (parsedmess[4].split(" ")[0] == "Checked"){
iconspan.className = "far fa-thumbs-up";
}
iconspan.innerHTML = "&nbsp;";
msage.appendChild(iconspan);
var message = parsedmess[4].split(" ");
message.shift();
msage.appendChild(document.createTextNode(message.join(" ")));
msage.appendChild(document.createTextNode(message.join(" ")));
var paste_path = parsedmess[5];
var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
var paste_path = parsedmess[5];
var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", url_to_saved_paste);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fa fa-search-plus";
action_icon_a.appendChild(action_icon_span);
var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", url_to_saved_paste);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fas fa-search-plus";
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a);
inspect.setAttribute("style", "text-align:center;");
inspect.appendChild(action_icon_a);
inspect.setAttribute("style", "text-align:center;");
tr.appendChild(time)
tr.appendChild(chan);
tr.appendChild(level);
tr.appendChild(scrpt);
tr.appendChild(src);
tr.appendChild(pdate);
tr.appendChild(nam);
tr.appendChild(msage);
tr.appendChild(inspect);
tr.appendChild(time)
tr.appendChild(chan);
tr.appendChild(level);
tr.appendChild(scrpt);
tr.appendChild(src);
tr.appendChild(pdate);
tr.appendChild(nam);
tr.appendChild(msage);
tr.appendChild(inspect);
if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked == true) {
tableBody.appendChild(tr);
}
if (tr.className == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) {
tableBody.appendChild(tr);
}
if (tr.className == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) {
tableBody.appendChild(tr);
};
if (chansplit[1] == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked == true) {
tableBody.appendChild(tr);
}
if (chansplit[1] == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) {
tableBody.appendChild(tr);
}
if (chansplit[1] == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) {
tableBody.appendChild(tr);
};
var sel = document.getElementById("log_select")
if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) {
while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){
tableBody.deleteRow(0);
}
var sel = document.getElementById("log_select")
if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) {
while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){
tableBody.deleteRow(0);
}
}
}
}
@ -294,7 +296,7 @@ function create_queue_table() {
document.getElementById("queueing").innerHTML = "";
var Tablediv = document.getElementById("queueing")
var table = document.createElement('TABLE')
table.className = "table table-bordered table-hover table-striped tableQueue";
table.className = "table table-bordered table-hover tableQueue";
var tableHead = document.createElement('THEAD')
var tableBody = document.createElement('TBODY')
@ -323,8 +325,8 @@ function create_queue_table() {
var td2 = document.createElement('TD');
td.appendChild(document.createTextNode("No running queues"));
td2.appendChild(document.createTextNode("Or no feed"));
td.className += " danger";
td2.className += " danger";
td.className += " table-danger";
td2.className += " table-danger";
tr.appendChild(td);
tr.appendChild(td2);
tableBody.appendChild(tr);
@ -345,11 +347,11 @@ function create_queue_table() {
// - j=2: LastProcessedPasteTime
// - j=3: Number of the module belonging in the same category
if (parseInt(glob_tabvar.row1[i][2]) > window.threshold_stucked_module && parseInt(glob_tabvar.row1[i][1]) > 2)
tr.className += " danger";
tr.className += " table-danger";
else if (parseInt(glob_tabvar.row1[i][1]) == 0)
tr.className += " warning";
tr.className += " table-disabled";
else
tr.className += " success";
tr.className += " table-success";
tableBody.appendChild(tr);
}
}
@ -474,29 +476,12 @@ function load_queues() {
g.updateOptions( { 'file': data } );
g2.updateOptions( { 'file': data2 } );
// TagCanvas.Reload('myCanvas');
}, interval);
}
});
};
refresh();
try {
var options = {
weight:true,
weightMode:"both",
noMouse:true,
textColour: '#2E9AFE'
}
TagCanvas.Start('myCanvas','',options);
TagCanvas.SetSpeed('myCanvas', [0.05, -0.15]);
} catch(e) {
// something went wrong, hide the canvas container
document.getElementById('myCanvasContainer').style.display = 'none';
}
}
function manage_undefined() {

41
var/www/templates/dashboard/menu_sidebar.html Normal file
View file

@ -0,0 +1,41 @@
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
<i class="fas fa-align-left"></i>
<span>Toggle Sidebar</span>
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
</nav>
<span id="side-bard-dashboard-content">
<div class="card">
<div class="card-header">
<i class="fas fa-tachometer-alt"></i> Total pastes since {{ default_minute }} min
</div>
<div class="card-body">
<div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
</div>
</div>
<div class="custom-control custom-switch ml-5 mt-2">
<input class="custom-control-input" id="button-toggle-queues" type="checkbox" checked>
<label class="custom-control-label" for="button-toggle-queues"><strong style="top: 3px; position: relative;">Display queues</strong></label>
</div>
<hr>
<div>
<div>
<table id="queue-color-legend" class="table table-borderless table-sm">
<tbody>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #e8e8e8;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
</tbody>
</table>
</div>
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
</div>
</span>
</div>

7
var/www/templates/nav_bar.html
View file

@ -10,7 +10,7 @@
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item">
<a class="nav-link mr-3" href="{{ url_for('dashboard.index') }}">Home</a>
<a class="nav-link mr-3" id="page-Dashboard" href="{{ url_for('dashboard.index') }}">Home</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" href="{{ url_for('PasteSubmit.PasteSubmit_page') }}" aria-disabled="true"><i class="fas fa-external-link-alt"></i> Submit</a>
@ -38,10 +38,11 @@
</li>
</ul>
<form class="form-inline my-2 my-lg-0 ml-auto justify-content-center">
<form class="form-inline my-2 my-lg-0 ml-auto justify-content-center" action="{{ url_for('searches.search') }}" method=POST>
<div class="d-flex flex-column">
<div>
<input class="form-control mr-sm-2" type="search" id="global_search" placeholder="Search" aria-label="Search" aria-describedby="advanced_search">
<input class="form-control mr-sm-2" type="search" id="global_search" name="query" placeholder="Search" aria-label="Search" aria-describedby="advanced_search">
<input type="hidden" name="index_name" class="form-control" value="0" placeholder="Index Name">
<button class="btn btn-outline-info my-2 my-sm-0" type="submit"><i class="fas fa-search"></i></button>
</div>
<small id="advanced_search" class="form-text"><a class="nav text-muted" href="#" aria-disabled="true">Advanced Search</a></small>