AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-29 17:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [flask] remove old modules + modules loader

Browse source
This commit is contained in:
terrtia 2024-11-27 16:10:44 +01:00
parent 14b0675330
commit 1c770988c5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
15 changed files with 452 additions and 694 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
var/www/modules/Flask_config.py → var/www/Flask_config.py
View file

42
var/www/Flask_server.py
View file

@ -14,11 +14,6 @@ from flask import Flask, render_template, jsonify, request, Request, Response, s
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
from flask_sock import Sock
import importlib
from os.path import join
sys.path.append('./modules/')
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
@ -37,6 +32,8 @@ import Flask_config
# Import Blueprint
from blueprints.root import root
from blueprints.dashboard import dashboard
from blueprints.ui_submit import PasteSubmit # TODO RENAME ME
from blueprints.crawler_splash import crawler_splash
from blueprints.correlation import correlation
from blueprints.languages_ui import languages_ui
@ -48,7 +45,6 @@ from blueprints.hunters import hunters
from blueprints.old_endpoints import old_endpoints
from blueprints.ail_2_ail_sync import ail_2_ail_sync
from blueprints.settings_b import settings_b
from blueprints.objects_objs import objects_objs
from blueprints.objects_cve import objects_cve
from blueprints.objects_decoded import objects_decoded
from blueprints.objects_subtypes import objects_subtypes
@ -124,6 +120,8 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
# ========= BLUEPRINT =========#
app.register_blueprint(root, url_prefix=baseUrl)
app.register_blueprint(dashboard, url_prefix=baseUrl)
app.register_blueprint(PasteSubmit, url_prefix=baseUrl)
app.register_blueprint(crawler_splash, url_prefix=baseUrl)
app.register_blueprint(correlation, url_prefix=baseUrl)
app.register_blueprint(languages_ui, url_prefix=baseUrl)
@ -136,7 +134,6 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl)
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
app.register_blueprint(settings_b, url_prefix=baseUrl)
app.register_blueprint(objects_cve, url_prefix=baseUrl)
app.register_blueprint(objects_objs, url_prefix=baseUrl)
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
app.register_blueprint(objects_title, url_prefix=baseUrl)
@ -175,37 +172,6 @@ def load_user(session_id):
return user
return None
# ========= HEADER GENERATION ======== DEPRECATED
# Get headers items that should be ignored (not displayed)
toIgnoreModule = set()
try:
with open('templates/ignored_modules.txt', 'r') as f:
lines = f.read().splitlines()
for line in lines:
toIgnoreModule.add(line)
except IOError:
pass
# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################
for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
sys.path.append(join(root))
# Ignore the module
curr_dir = root.split('/')[1]
if curr_dir in toIgnoreModule:
continue
for name in files:
module_name = root.split('/')[-2]
if name.startswith('Flask_') and name.endswith('.py'):
if name == 'Flask_config.py':
continue
name = name.strip('.py')
importlib.import_module(name)
# ========= JINJA2 FUNCTIONS ========
def list_len(s):

0
var/www/modules/Role_Manager.py → var/www/Role_Manager.py
View file

27
var/www/blueprints/objects_objs.py → var/www/blueprints/dashboard.py
View file

@ -7,9 +7,11 @@
import os
import sys
import json
import time
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context
import json # TODO REMOVE ME
from flask import render_template, Response, request, Blueprint
from flask_login import login_required, current_user
# Import Role_Manager
@ -21,21 +23,23 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib import ail_stats
# TODO STATS
# from lib import ail_updates
# ============ BLUEPRINT ============
objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
dashboard = Blueprint('dashboard', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/dashboard'))
# ============ VARIABLES ============
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
# ============ FUNCTIONS ============
@objects_objs.route("/objects", methods=['GET'])
# ============= ROUTES ==============
@dashboard.route("/", methods=['GET'])
@login_required
@login_read_only
def objects():
def index():
user_org = current_user.get_org()
user_id = current_user.get_user_id()
nb_objects = ail_stats.get_nb_objs_dashboard()
@ -43,11 +47,6 @@ def objects():
crawlers_stats = ail_stats.get_crawlers_stats()
trackers = ail_stats.get_tracked_objs_dashboard(user_org, user_id)
tagged_objs = ail_stats.get_tagged_objs_dashboard()
return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard,
return render_template("dashboard.html", feeders_dashboard=feeders_dashboard,
nb_objects=nb_objects, trackers=trackers, tagged_objs=tagged_objs,
bootstrap_label=bootstrap_label, crawlers_stats=crawlers_stats)
# ============= ROUTES ==============

5
var/www/modules/PasteSubmit/Flask_PasteSubmit.py → var/www/blueprints/ui_submit.py
View file

@ -50,7 +50,7 @@ text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000)
file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/submit'))
# ============ Validators ============
def limit_content_length():
@ -296,6 +296,3 @@ def submit_status():
else:
return 'INVALID UUID'
# ========= REGISTRATION =========
app.register_blueprint(PasteSubmit, url_prefix=baseUrl)

1
var/www/modules/PasteSubmit/templates/header_PasteSubmit.html
View file

@ -1 +0,0 @@
<li id='page-PasteSubmit'><a href="{{ url_for('PasteSubmit.PasteSubmit_page') }}"><i class="glyphicon glyphicon-new-window white"></i> Submit Paste </a></li>

199
var/www/modules/dashboard/Flask_dashboard.py
View file

@ -1,199 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the dashboard page
'''
import json
import os
import sys
import datetime
import time
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, stream_with_context
from Role_Manager import login_admin, login_read_only
from flask_login import login_required
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_queues
from lib import ail_updates
from packages.Date import Date
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl
r_serv_log = Flask_config.r_serv_log
max_dashboard_logs = Flask_config.max_dashboard_logs
dashboard = Blueprint('dashboard', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def event_stream():
pubsub = r_serv_log.pubsub()
pubsub.psubscribe("Script" + '.*')
try:
for msg in pubsub.listen():
mtype = msg['type']
pattern = msg['pattern']
channel = msg['channel']
data = msg['data']
msg = {'channel': channel, 'type': mtype, 'pattern': pattern, 'data': data}
level = (msg['channel']).split('.')[1]
if msg['type'] == 'pmessage' and level != "DEBUG":
yield 'data: %s\n\n' % json.dumps(msg)
except GeneratorExit:
print("Generator Exited")
pubsub.unsubscribe()
def event_stream_dashboard():
try:
while True:
data = {'queues': get_queues()}
yield f'data: {json.dumps(data)}\n\n'
time.sleep(1)
except GeneratorExit:
print("Generator dashboard Exited")
pass
def get_queues():
# We may want to put the llen in a pipeline to do only one query.
return ail_queues.get_modules_queues_stats()
def get_date_range(date_from, num_day):
date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2))
date_list = []
for i in range(0, num_day+1):
new_date = date.substract_day(i)
date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}')
return date_list
def dashboard_alert(log):
# check if we need to display this log
if len(log) > 50:
date = log[1:5]+log[6:8]+log[9:11]
utc_str = log[1:20]
log = log[46:].split(';')
if len(log) == 6:
date_time = datetime_from_utc_to_local(utc_str)
path = url_for('investigations_b.get_object_gid', gid=log[5])
res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
'paste': log[3], 'message': log[4], 'path': path}
return res
else:
return False
else:
return False
def datetime_from_utc_to_local(utc_str):
utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S')
now_timestamp = time.time()
offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
local_time_str = (utc_datetime + offset).strftime('%H:%M:%S')
return local_time_str
# ============ ROUTES ============
@dashboard.route("/_logs")
@login_required
@login_read_only
def logs():
return flask.Response(event_stream(), mimetype="text/event-stream")
@dashboard.route("/_dashboard")
@login_required
@login_read_only
def _dashboard():
return flask.Response(stream_with_context(event_stream_dashboard()), content_type="text/event-stream")
@dashboard.route("/_get_last_logs_json")
@login_required
@login_read_only
def get_last_logs_json():
date = datetime.datetime.now().strftime("%Y%m%d")
max_day_search = 6
day_search = 0
warning_found = 0
warning_to_found = max_dashboard_logs
last_logs = []
date_range = get_date_range(date, max_day_search)
while max_day_search != day_search and warning_found != warning_to_found:
filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log'
filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
try:
with open(filename_log, 'r') as f:
lines = f.read().splitlines()
curr_index = -1
while warning_found != warning_to_found:
try:
# get lasts warning logs
log_warn = dashboard_alert(lines[curr_index])
if log_warn != False:
last_logs.append(log_warn)
warning_found = warning_found + 1
curr_index = curr_index - 1
except IndexError:
# check previous warning log file
day_search = day_search + 1
break
except FileNotFoundError:
# check previous warning log file
day_search = day_search + 1
return jsonify(list(reversed(last_logs)))
@dashboard.route("/_stuff", methods=['GET'])
@login_required
@login_read_only
def stuff():
return jsonify(row1=get_queues())
# TODO: ADD UPDATE NOTE BY USER
@dashboard.route("/")
@login_required
@login_read_only
def index():
default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")
log_select = {10, 25, 50, 100}
log_select.add(max_dashboard_logs)
log_select = list(log_select)
log_select.sort()
# Check if update in progress
background_update = False
update_message = ''
if ail_updates.is_update_background_running():
background_update = True
update_message = ail_updates.AILBackgroundUpdate(ail_updates.get_update_background_version()).get_message()
return render_template("index.html", default_minute = default_minute,
threshold_stucked_module=threshold_stucked_module,
log_select=log_select, selected=max_dashboard_logs,
background_update=background_update, update_message=update_message)
# ========= REGISTRATION =========
app.register_blueprint(dashboard, url_prefix=baseUrl)

42
var/www/modules/dashboard/templates/Queue_live_Monitoring.html
View file

@ -1,42 +0,0 @@
<html>
<head>
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
</head>
<body>
<title>WordsTrend</title>
<script type="text/javascript">
$(document).ready(function () {
var data = [];
var t = new Date();
for (var i = 10; i >= 0; i--) {
var x = new Date(t.getTime() - i * 1000);
data.push([x, 0]);
}
var g = new Dygraph(document.getElementById("Graph"), data,
{
drawPoints: true,
showRoller: true,
//valueRange: [0.0, 1.2],
legend: 'always',
title: 'Pastes Queues Monitoring',
ylabel: 'Numbers of pastes waiting',
xlabel: 'Time',
});
// It sucks that these things aren't objects, and we need to store state in window.
window.intervalId = setInterval(function() {
var x = new Date(); // current time
var y = 6;
data.push([x, y]);
g.updateOptions( { 'file': data } );
}, 1000);
}
);
</script>
<div id="Graph" style="width:100%; height:100%;"></div>
</body>
</html>

340
var/www/modules/dashboard/templates/index.html
View file

@ -1,340 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Analysis Information Leak framework Dashboard</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
data-urllog="{{ url_for('dashboard.logs') }}">
</script>
<style>
.tableQueue tbody tr td,
.tableQueue tbody tr th,
.tableQueue thead tr td,
.tableQueue thead tr th{
padding: 1px;
}
tr.table-disabled {
background-color: #f2f2f2;
}
tr.table-log-warning {
background-color: #ccfae3;
}
</style>
<script>
window.default_minute = {{ default_minute }};
</script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'dashboard/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{%if background_update%}
<div class="alert alert-warning alert-dismissible fade show my-2" role="alert">
<strong>Warning!</strong> An Update is running on the background. Some informations can be <strong>missing from the UI.</strong>
<strong>Updating: {{update_message}}</strong>
(<a href="{{ url_for('settings_b.settings_page') }}">Check Update Status</a>)
<button type="button" class="close" data-dismiss="alert" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
{%endif%}
<!-- TODO: Add users messages -->
{# {% include 'dashboard/update_modal.html' %}#}
<div class="row my-2">
<div class="col-xl-6">
<div class="card">
<div class="card-header">
<i class="far fa-chart-bar"></i> Feeder(s) Monitor:
</div>
<div class="card-body" id="panelbody"> {#style="height:420px;"#}
<strong>Processed items</strong>
<div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
</div>
</div>
</div>
<div class="col-xl-6">
{# <div class="card">#}
{# <div class="card-header">#}
{# <i class="far fa-chart-bar"></i> Queues Monitor#}
{# </div>#}
{# <div class="card-body">#}
{# <div class="" id="Graph" style="height:195px; width:88%;"></div>#}
{# <div style="height:10px;"></div>#}
{# <div class="" id="Graph2" style="height:195px; width:88%;"></div>#}
{# </div>#}
{##}
{# </div>#}
</div>
</div>
<div class="card">
<div class="card-header">
<i class="fas fa-clipboard-list"></i> Logs
<div class="float-right">
<label style="padding-bottom:2px;">
<select class="form-control input-sm" id="log_select">
{% for log_selection in log_select %}
{% if log_selection == selected %}
<option value="{{ log_selection }}" selected>{{ log_selection }}</option>
{% else %}
<option value="{{ log_selection }}">{{ log_selection }}</option>
{% endif %}
{% endfor %}
</select>
</label>
<input id="checkbox_log_info" type="checkbox" value="INFO"> INFO
<input id="checkbox_log_warning" type="checkbox" value="WARNING" checked="true"> WARNING
<input id="checkbox_log_critical" type="checkbox" value="CRITICAL" checked="true"> CRITICAL
</div>
</div>
<div class="card-body">
<table class="table table-hover table-sm" id="table_log">
<thead>
<tr>
<th>Time</th>
<th>Channel</th>
<th>Level</th>
<th>Script Name</th>
<th>Source</th>
<th>Date</th>
<th>Paste name</th>
<th>Message</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="tab_body">
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var url_showSavedPath = "{{ url_for('investigations_b.get_object_gid') }}";
</script>
<script>
$("#page-Dashboard").addClass("active");
var tableBody = document.getElementById('tab_body')
$.getJSON("{{ url_for('dashboard.get_last_logs_json') }}", function(data) {
data.forEach(function (d) {
var tr = document.createElement('TR')
var time = document.createElement('TD')
var chan = document.createElement('TD')
var level = document.createElement('TD')
var scrpt = document.createElement('TD')
var src = document.createElement('TD')
var pdate = document.createElement('TD')
var nam = document.createElement('TD')
var msage = document.createElement('TD')
var inspect = document.createElement('TD')
// console.log(d)
tr.className = "table-log-warning";
time.appendChild(document.createTextNode(d.time))
chan.appendChild(document.createTextNode('Script'))
level.appendChild(document.createTextNode('WARNING'))
scrpt.appendChild(document.createTextNode(d.script))
src.appendChild(document.createTextNode(d.domain))
pdate.appendChild(document.createTextNode(d.date_paste))
nam.appendChild(document.createTextNode(d.paste))
var iconspan = document.createElement('SPAN');
var message = d.message.split(" ")
if (message[0] == "Detected"){
iconspan.className = "fas fa-eye";
}
else if (message[0] == "Checked"){
iconspan.className = "far fa-thumbs-up";
}
iconspan.innerHTML = "&nbsp;";
msage.appendChild(iconspan);
msage.appendChild(document.createTextNode(message.join(" ")));
var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", d.path);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fas fa-search-plus";
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a)
inspect.setAttribute("style", "text-align:center;");
tr.appendChild(time)
tr.appendChild(chan);
tr.appendChild(level);
tr.appendChild(scrpt);
tr.appendChild(src);
tr.appendChild(pdate);
tr.appendChild(nam);
tr.appendChild(msage);
tr.appendChild(inspect);
tableBody.appendChild(tr);
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side-bard-dashboard-content').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side-bard-dashboard-content').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function create_queue_table(data) {
document.getElementById("queueing").innerHTML = "";
let Tablediv = document.getElementById("queueing")
let table = document.createElement('TABLE')
table.className = "table table-bordered table-hover tableQueue";
let tableHead = document.createElement('THEAD')
let tableBody = document.createElement('TBODY')
table.appendChild(tableHead);
table.appendChild(tableBody);
let heading = new Array();
heading[0] = "Queue Name.PID"
heading[1] = "Amount"
let tr = document.createElement('TR');
tableHead.appendChild(tr);
for (i = 0; i < heading.length; i++) {
let th = document.createElement('TH')
if (heading[i] === "Amount") {
th.width = '50';
} else {
th.width = '100';
}
th.appendChild(document.createTextNode(heading[i]));
tr.appendChild(th);
}
if ((data).length === 0) {
let tr = document.createElement('TR');
let td = document.createElement('TD');
let td2 = document.createElement('TD');
td.appendChild(document.createTextNode("No running queues"));
td2.appendChild(document.createTextNode("Or no feed"));
td.className += " table-danger";
td2.className += " table-danger";
tr.appendChild(td);
tr.appendChild(td2);
tableBody.appendChild(tr);
}
else {
for(i = 0; i < (data).length;i++){
let tr = document.createElement('TR')
for(j = 0; j < 2; j++){
let td = document.createElement('TD')
let moduleNum = j == 0 ? "." + data[i][3] : "";
td.appendChild(document.createTextNode(data[i][j] + moduleNum));
tr.appendChild(td)
}
// Used to decide the color of the row
// We have data[][j] with:
// - j=0: ModuleName
// - j=1: queueLength
// - j=2: LastProcessedPasteTime
// - j=3: Number of the module belonging in the same category
if (data[i][3]==="Not Launched")
tr.className += " bg-danger text-white";
else if (parseInt(data[i][2]) > window.threshold_stucked_module && parseInt(data[i][1]) > 2)
tr.className += " table-danger";
else if (parseInt(data[i][1]) === 0)
tr.className += " table-disabled";
else
tr.className += " table-success";
tableBody.appendChild(tr);
}
}
Tablediv.appendChild(table);
}
//
//
var event_queues
$(document).ready(function() {
// Hide Browser Error: connection interrupted while the page was loading
$(window).on("beforeunload", function() {
event_queues.close();
source.close();
});
event_queues = new EventSource("{{ url_for('dashboard._dashboard') }}");
event_queues.onmessage = function(event) {
let data = JSON.parse(event.data);
create_queue_table(data['queues'])
};
event_queues.onerror = function(event) {
console.error('EventSource failed:', event);
event_queues.close();
};
});
</script>
</html>

434
var/www/templates/dashboard/dashboard.html Normal file
View file

@ -0,0 +1,434 @@
<!DOCTYPE html>
<html>
<head>
<title>Objects - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/echarts.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<style>
.popover {
max-width: none;
width: 500px;
}
.icon-button {
position: relative;
display: inline-block;
border: none;
background: none;
{#padding: 0;#}
margin: 0;
cursor: pointer;
outline: inherit;
transition: color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;
}
.icon-wrapper {
position: relative;
display: inline-block;
}
.notification-badge {
position: absolute;
top: -10px;
right: -10px;
font-size: 14px;
background: forestgreen;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
<div class="col-12 col-xl-6">
<div id="feeders_dashboard" style="width: 100%; height:600px;"></div>
<div class="card mt-1 mb-4">
<div class="card-header text-white bg-dark">
<div class="row">
<div class="col-1">
<i class="fas fa-user-secret fa-2x"></i>
</div>
<div class="col-5">
<div class="row">
<div class="col-6">
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&up=True" class="badge badge-success" id="stat_crawler_onion_domain_up">{{ crawlers_stats['onion']['up'] }}</a> Up
</div>
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&down=True" class="badge badge-danger" id="stat_crawler_onion_domain_down">{{ crawlers_stats['onion']['down'] }}</a> Down
</div>
</div>
<div class="col-6">
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&up=True&down=True" class="badge badge-success" id="stat_crawler_onion_total">{{ crawlers_stats['onion']['crawled'] }}</a> Crawled
</div>
<div>
<span class="badge badge-warning" id="stat_crawler_onion_queue">{{ crawlers_stats['onion']['queue'] }}</span> Queue
</div>
</div>
</div>
</div>
<div class="col-1">
<i class="fab fa-html5 fa-2x"></i>
</div>
<div class="col-5">
<div class="row">
<div class="col-6">
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&up=True" class="badge badge-success" id="stat_crawler_regular_domain_up">{{ crawlers_stats['web']['up'] }}</a> UP
</div>
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&down=True" class="badge badge-danger" id="stat_crawler_regular_domain_down">{{ crawlers_stats['web']['down'] }}</a> DOWN
</div>
</div>
<div class="col-6">
<div>
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&up=True&down=True" class="badge badge-success" id="stat_crawler_regular_total">{{ crawlers_stats['web']['crawled'] }}</a> Crawled
</div>
<div>
<span class="badge badge-warning" id="stat_crawler_regular_queue">{{ crawlers_stats['web']['queue'] }}</span> Queue
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-12 col-xl-6 mt-4">
<div class="row">
{% for obj_type in nb_objects %}
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
{% with type=obj_type, name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %}
{% include 'objects/block_obj_button.html' %}
{% endwith %}
</div>
{% endfor %}
</div>
</div>
</div>
<div class="row">
<div class="col-12 col-xl-6">
<table class="table">
<thead>
<tr>
<th><i class="fa-solid fa-crosshairs"></i></th>
<th>tracker</th>
<th class="text-center"><i class="fas fa-cube"></i></th>
<th>Time</th>
</tr>
</thead>
<tbody id="tracked_objs">
{% for tracker in trackers %}
<tr>
<td>{{ tracker['type'] }}</td>
<td style="word-break: break-all;">
<a href="{{ url_for('hunters.show_tracker') }}?uuid={{ tracker['uuid'] }}">{{ tracker['tracked'] }}</a>
</td>
<td class="text-center">
<a href="{{ tracker['obj']['link'] }}" onmouseenter="show_obj_tooltip(this, '{{ tracker['obj']['gid'] }}')" onmouseleave="hide_obj_tooltip(this)">
<svg height="26" width="26" style="pointer-events:none;">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ tracker['obj']['icon']['color'] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ tracker['obj']['icon']['style'] }}" font-size="16px">{{ tracker['obj']['icon']['icon'] }}</text>
</g>
</svg>
</a>
</td>
<td>{{ tracker['timestamp'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="col-12 col-xl-6">
<table class="table">
<thead>
<tr>
<th class="text-center"><i class="fas fa-cube"></i></th>
<th>ID</th>
<th><i class="fas fa-tag"></i> Tags</th>
<th>Time</th>
</tr>
</thead>
<tbody id="t_tags_objs">
{% for t_obj in tagged_objs %}
<tr>
<td>
<a href="{{ t_obj['link'] }}" onmouseenter="show_obj_tooltip(this, '{{ t_obj['gid'] }}')" onmouseleave="hide_obj_tooltip(this)">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ t_obj['icon']['color'] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ t_obj['icon']['style'] }}" font-size="16px">{{ t_obj['icon']['icon'] }}</text>
</g>
</svg>
</a>
</td>
<td style="word-break: break-all;">
<a href="{{ t_obj['link'] }}">{{ t_obj['id'] }}</a>
</td>
<td>
{% for tag in t_obj['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</td>
<td>{{ t_obj['date_tag'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_dashboard_{{obj_type}}").addClass("active");
});
var btn_selector = {
{% for obj_type in nb_objects %}
'{{ obj_type }}': $("#obj_btn_{{ obj_type }}"),
{% endfor %}
}
</script>
<script>
var tags_colors = ['primary', 'success', 'danger', 'warning', 'info'];
var url_obj_description = "{{ url_for('correlation.get_description') }}?object_id=";
// Init Dashboard
var feederChart = echarts.init(document.getElementById('feeders_dashboard'));
window.addEventListener('resize', function() {
feederChart.resize();
});
var optionFeeder;
var maxDataLength = 21;
var feeders_names = [{% for feeder in feeders_dashboard['data'] %}'{{ feeder }}', {% endfor %}]
var feeders_xaxis = [{% for date in feeders_dashboard['dates'] %}'{{ date }}', {% endfor %}]
var feeders_data = {
{% for feeder in feeders_dashboard['data'] %}
{{ feeder }}: {{ feeders_dashboard['data'][feeder] }},
{% endfor %}
}
optionFeeder = {
title: {
text: 'Feeders'
},
tooltip: {
trigger: 'axis',
axisPointer: {
type: 'cross',
label: {
backgroundColor: '#6a7985'
}
}
},
legend: {
data: feeders_names
},
grid: {
left: '3%',
right: '4%',
bottom: '3%',
containLabel: true
},
xAxis: [
{
type: 'category',
boundaryGap: false,
data: feeders_xaxis
}
],
yAxis: [
{
type: 'value'
}
],
series: [
{% for feeder in feeders_dashboard['data'] %}
{
name: '{{ feeder }}',
type: 'line',
stack: 'Total',
areaStyle: {},
emphasis: {
focus: 'series'
},
data: feeders_data['{{ feeder }}']
},
{% endfor %}
]
};
optionFeeder && feederChart.setOption(optionFeeder);
{# TODO UPDATE NEW FEEDER NAME#}
function updateFeederChart(data) {
let new_date = data['date']
feeders_xaxis.push(new_date)
feeders_xaxis.shift()
for (const f_name in data['data']) {
if (f_name in feeders_data) {
feeders_data[f_name].push(data['data'][f_name])
feeders_data[f_name].shift()
} else {
let fdata = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, data['data'][f_name]]
feeders_names.push(f_name)
feeders_data[f_name] = fdata
// add new feeder
optionFeeder.legend.data.push(f_name)
optionFeeder.series.push({name: f_name, type: 'line', stack: 'Total', areaStyle: {}, emphasis: {focus: 'series'}, data: feeders_data[f_name]})
}
}
feederChart.setOption(optionFeeder)
}
function updateNbObjects(data) {
for (const obj_type in data) {
btn_selector[obj_type].text(data[obj_type]);
}
}
function updateCrawlerStats(data) {
$("#stat_crawler_onion_domain_up").text(data['onion']['up']);
$("#stat_crawler_onion_domain_down").text(data['onion']['down']);
$("#stat_crawler_onion_total").text(data['onion']['crawled']);
$("#stat_crawler_onion_queue").text(data['onion']['queue']);
$("#stat_crawler_regular_domain_up").text(data['web']['up']);
$("#stat_crawler_regular_domain_down").text(data['web']['down']);
$("#stat_crawler_regular_total").text(data['web']['crawled']);
$("#stat_crawler_regular_queue").text(data['web']['queue']);
}
///////
function create_obj_svg(container, obj_gid, url, color, fa_style, icon) {
var svg_obj = '<svg height="26" width="26"> <g class="nodes"> <circle cx="13" cy="13" r="13" fill="' + color + '"></circle> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="' + fa_style + '" font-size="16px">' + icon + '</text> </g> </svg>';
var obj_link = $('<a>');
obj_link.on('mouseenter', function () {
show_obj_tooltip(this, obj_gid);
});
obj_link.on('mouseleave', function () {
hide_obj_tooltip(this);
});
obj_link.attr('href', url);
obj_link.html(svg_obj);
container.append(obj_link);
return container
}
function create_tags_badges(container, tags) {
var tspan = '';
for (const i in tags) {
tspan = tspan + '<span class="badge badge-' + 'primary' + '">' + sanitize_text(tags[i]) + '</span>'
}
return tspan;
}
function update_tags_obj_dashboard(data) {
// delete table
var tbody = $("#t_tags_objs");
var ntd;
tbody.empty();
for (const elem in data) {
let obj = data[elem];
var row = $('<tr></tr>');
ntd = $('<td></td>');
row.append(create_obj_svg(ntd, obj['gid'], obj.link, obj['icon']['color'], obj['icon']['style'], obj['icon']['icon']))
row.append('<td style="word-break: break-all;"><a href="'+ obj['link'] + '">' + sanitize_text(obj['id']) + '</a></td>');
ntd = $('<td></td>');
row.append(create_tags_badges(ntd, obj['tags']));
row.append('<td>' + obj['date_tag'] + '</td>');
tbody.append(row);
{# {% for tag in t_obj['tags'] %}#}
{# <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>#}
{# {% endfor %}#}
{#</td>#}
}
}
function update_tracked_objs_dashboard(data) {
// delete table
var tbody = $("#tracked_objs");
var ntd;
tbody.empty();
for (const elem in data) {
let tracker = data[elem];
var row = $('<tr></tr>');
row.append('<td>' + tracker['type'] + '</td>');
row.append('<td style="word-break: break-all;"><a href="{{ url_for('hunters.show_tracker') }}?uuid=' + tracker['uuid'] + '">' + sanitize_text(tracker['tracked']) + '</a></td>');
ntd = $('<td class="text-center"></td>');
row.append(create_obj_svg(ntd, tracker['obj']['gid'], tracker['obj']['link'], tracker['obj']['icon']['color'], tracker['obj']['icon']['style'], tracker['obj']['icon']['icon']));
row.append('<td>' + tracker['timestamp'] + '</td>');
tbody.append(row);
}
}
// WebSocket
var socket = new WebSocket("{{ url_for('ws_dashboard') }}");
socket.wsocket = function(event) {
console.log("WebSocket connection opened:", event);
};
socket.onmessage = function(event) {
$("[data-toggle='popover']").popover('destroy');
let data = JSON.parse(event.data);
updateFeederChart(data['feeders']);
updateNbObjects(data['objs']);
updateCrawlerStats(data['crawler']);
update_tracked_objs_dashboard(data['trackers']);
update_tags_obj_dashboard(data['tags']);
};
socket.onerror = function(error) {
console.error('WebSocket error:', error);
};
socket.onclose = function(event) {
console.log('WebSocket connection closed:', event);
};
</script>
</body>
</html>

41
var/www/templates/dashboard/menu_sidebar.html
View file

@ -1,41 +0,0 @@
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
<i class="fas fa-align-left"></i>
<span>Toggle Sidebar</span>
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
</nav>
<span id="side-bard-dashboard-content">
<div class="card">
<div class="card-header">
<i class="fas fa-tachometer-alt"></i> Total pastes since {{ default_minute }} min
</div>
<div class="card-body">
<div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
</div>
</div>
<div class="custom-control custom-switch ml-5 mt-2">
<input class="custom-control-input" id="button-toggle-queues" type="checkbox" checked>
<label class="custom-control-label" for="button-toggle-queues"><strong style="top: 3px; position: relative;">Display queues</strong></label>
</div>
<hr>
<div>
<div>
<table id="queue-color-legend" class="table table-borderless table-sm">
<tbody>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #e8e8e8;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
</tbody>
</table>
</div>
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
</div>
</span>
</div>

13
var/www/templates/header_base.html
View file

@ -1,13 +0,0 @@
<div class="navbar-header">
<div class="nav navbar-nav">
<!--dashboard-->
<!--PasteSubmit-->
<!--Tags-->
<!--terms-->
<!--browsepastes-->
<!--trendingcharts-->
<!--trendingmodules-->
<!--sentiment-->
<!--insert here-->
</div>
</div>

1
var/www/templates/header_base_template.html
View file

@ -1 +0,0 @@
<li id='page-MODULENAME'><a href="{{ url_for('MODULENAME.MODULENAME_page') }}"><i class="glyphicon glyphicon-new-window"></i> MODULENAME </a></li>

1
var/www/templates/ignored_modules.txt
View file

@ -1 +0,0 @@
rawSkeleton

0
var/www/modules/PasteSubmit/templates/submit_items.html → var/www/templates/submit/submit_items.html
View file