mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-30 01:37:17 +00:00
chg: [flask] remove old modules + modules loader
This commit is contained in:
terrtia
parent
14b0675330
commit
1c770988c5
15 changed files with 452 additions and 694 deletions
|
|
@ -14,11 +14,6 @@ from flask import Flask, render_template, jsonify, request, Request, Response, s
|
||||||
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
|
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
|
||||||
from flask_sock import Sock
|
from flask_sock import Sock
|
||||||
|
|
||||||
import importlib
|
|
||||||
from os.path import join
|
|
||||||
|
|
||||||
sys.path.append('./modules/')
|
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
# Import Project packages
|
# Import Project packages
|
||||||
|
|
@ -37,6 +32,8 @@ import Flask_config
|
||||||
|
|
||||||
# Import Blueprint
|
# Import Blueprint
|
||||||
from blueprints.root import root
|
from blueprints.root import root
|
||||||
|
from blueprints.dashboard import dashboard
|
||||||
|
from blueprints.ui_submit import PasteSubmit # TODO RENAME ME
|
||||||
from blueprints.crawler_splash import crawler_splash
|
from blueprints.crawler_splash import crawler_splash
|
||||||
from blueprints.correlation import correlation
|
from blueprints.correlation import correlation
|
||||||
from blueprints.languages_ui import languages_ui
|
from blueprints.languages_ui import languages_ui
|
||||||
|
|
@ -48,7 +45,6 @@ from blueprints.hunters import hunters
|
||||||
from blueprints.old_endpoints import old_endpoints
|
from blueprints.old_endpoints import old_endpoints
|
||||||
from blueprints.ail_2_ail_sync import ail_2_ail_sync
|
from blueprints.ail_2_ail_sync import ail_2_ail_sync
|
||||||
from blueprints.settings_b import settings_b
|
from blueprints.settings_b import settings_b
|
||||||
from blueprints.objects_objs import objects_objs
|
|
||||||
from blueprints.objects_cve import objects_cve
|
from blueprints.objects_cve import objects_cve
|
||||||
from blueprints.objects_decoded import objects_decoded
|
from blueprints.objects_decoded import objects_decoded
|
||||||
from blueprints.objects_subtypes import objects_subtypes
|
from blueprints.objects_subtypes import objects_subtypes
|
||||||
|
|
@ -124,6 +120,8 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
|
||||||
|
|
||||||
# ========= BLUEPRINT =========#
|
# ========= BLUEPRINT =========#
|
||||||
app.register_blueprint(root, url_prefix=baseUrl)
|
app.register_blueprint(root, url_prefix=baseUrl)
|
||||||
|
app.register_blueprint(dashboard, url_prefix=baseUrl)
|
||||||
|
app.register_blueprint(PasteSubmit, url_prefix=baseUrl)
|
||||||
app.register_blueprint(crawler_splash, url_prefix=baseUrl)
|
app.register_blueprint(crawler_splash, url_prefix=baseUrl)
|
||||||
app.register_blueprint(correlation, url_prefix=baseUrl)
|
app.register_blueprint(correlation, url_prefix=baseUrl)
|
||||||
app.register_blueprint(languages_ui, url_prefix=baseUrl)
|
app.register_blueprint(languages_ui, url_prefix=baseUrl)
|
||||||
|
|
@ -136,7 +134,6 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl)
|
||||||
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
|
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
|
||||||
app.register_blueprint(settings_b, url_prefix=baseUrl)
|
app.register_blueprint(settings_b, url_prefix=baseUrl)
|
||||||
app.register_blueprint(objects_cve, url_prefix=baseUrl)
|
app.register_blueprint(objects_cve, url_prefix=baseUrl)
|
||||||
app.register_blueprint(objects_objs, url_prefix=baseUrl)
|
|
||||||
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
|
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
|
||||||
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
|
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
|
||||||
app.register_blueprint(objects_title, url_prefix=baseUrl)
|
app.register_blueprint(objects_title, url_prefix=baseUrl)
|
||||||
|
|
@ -175,37 +172,6 @@ def load_user(session_id):
|
||||||
return user
|
return user
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# ========= HEADER GENERATION ======== DEPRECATED
|
|
||||||
|
|
||||||
|
|
||||||
# Get headers items that should be ignored (not displayed)
|
|
||||||
toIgnoreModule = set()
|
|
||||||
try:
|
|
||||||
with open('templates/ignored_modules.txt', 'r') as f:
|
|
||||||
lines = f.read().splitlines()
|
|
||||||
for line in lines:
|
|
||||||
toIgnoreModule.add(line)
|
|
||||||
|
|
||||||
except IOError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################
|
|
||||||
for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
|
|
||||||
sys.path.append(join(root))
|
|
||||||
|
|
||||||
# Ignore the module
|
|
||||||
curr_dir = root.split('/')[1]
|
|
||||||
if curr_dir in toIgnoreModule:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for name in files:
|
|
||||||
module_name = root.split('/')[-2]
|
|
||||||
if name.startswith('Flask_') and name.endswith('.py'):
|
|
||||||
if name == 'Flask_config.py':
|
|
||||||
continue
|
|
||||||
name = name.strip('.py')
|
|
||||||
importlib.import_module(name)
|
|
||||||
|
|
||||||
|
|
||||||
# ========= JINJA2 FUNCTIONS ========
|
# ========= JINJA2 FUNCTIONS ========
|
||||||
def list_len(s):
|
def list_len(s):
|
||||||
|
|
|
||||||
|
|
@ -7,9 +7,11 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import json
|
import time
|
||||||
|
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context
|
import json # TODO REMOVE ME
|
||||||
|
|
||||||
|
from flask import render_template, Response, request, Blueprint
|
||||||
from flask_login import login_required, current_user
|
from flask_login import login_required, current_user
|
||||||
|
|
||||||
# Import Role_Manager
|
# Import Role_Manager
|
||||||
|
|
@ -21,21 +23,23 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
from lib import ail_stats
|
from lib import ail_stats
|
||||||
|
|
||||||
|
# TODO STATS
|
||||||
|
# from lib import ail_updates
|
||||||
|
|
||||||
# ============ BLUEPRINT ============
|
# ============ BLUEPRINT ============
|
||||||
objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
|
dashboard = Blueprint('dashboard', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/dashboard'))
|
||||||
|
|
||||||
|
|
||||||
# ============ VARIABLES ============
|
# ============ VARIABLES ============
|
||||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||||
|
|
||||||
|
|
||||||
# ============ FUNCTIONS ============
|
# ============ FUNCTIONS ============
|
||||||
@objects_objs.route("/objects", methods=['GET'])
|
|
||||||
|
# ============= ROUTES ==============
|
||||||
|
|
||||||
|
@dashboard.route("/", methods=['GET'])
|
||||||
@login_required
|
@login_required
|
||||||
@login_read_only
|
@login_read_only
|
||||||
def objects():
|
def index():
|
||||||
user_org = current_user.get_org()
|
user_org = current_user.get_org()
|
||||||
user_id = current_user.get_user_id()
|
user_id = current_user.get_user_id()
|
||||||
nb_objects = ail_stats.get_nb_objs_dashboard()
|
nb_objects = ail_stats.get_nb_objs_dashboard()
|
||||||
|
|
@ -43,11 +47,6 @@ def objects():
|
||||||
crawlers_stats = ail_stats.get_crawlers_stats()
|
crawlers_stats = ail_stats.get_crawlers_stats()
|
||||||
trackers = ail_stats.get_tracked_objs_dashboard(user_org, user_id)
|
trackers = ail_stats.get_tracked_objs_dashboard(user_org, user_id)
|
||||||
tagged_objs = ail_stats.get_tagged_objs_dashboard()
|
tagged_objs = ail_stats.get_tagged_objs_dashboard()
|
||||||
return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard,
|
return render_template("dashboard.html", feeders_dashboard=feeders_dashboard,
|
||||||
nb_objects=nb_objects, trackers=trackers, tagged_objs=tagged_objs,
|
nb_objects=nb_objects, trackers=trackers, tagged_objs=tagged_objs,
|
||||||
bootstrap_label=bootstrap_label, crawlers_stats=crawlers_stats)
|
bootstrap_label=bootstrap_label, crawlers_stats=crawlers_stats)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ============= ROUTES ==============
|
|
||||||
|
|
||||||
|
|
@ -50,7 +50,7 @@ text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000)
|
||||||
file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
|
file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
|
||||||
allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
|
allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
|
||||||
|
|
||||||
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
|
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/submit'))
|
||||||
|
|
||||||
# ============ Validators ============
|
# ============ Validators ============
|
||||||
def limit_content_length():
|
def limit_content_length():
|
||||||
|
|
@ -296,6 +296,3 @@ def submit_status():
|
||||||
else:
|
else:
|
||||||
return 'INVALID UUID'
|
return 'INVALID UUID'
|
||||||
|
|
||||||
|
|
||||||
# ========= REGISTRATION =========
|
|
||||||
app.register_blueprint(PasteSubmit, url_prefix=baseUrl)
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
<li id='page-PasteSubmit'><a href="{{ url_for('PasteSubmit.PasteSubmit_page') }}"><i class="glyphicon glyphicon-new-window white"></i> Submit Paste </a></li>
|
|
||||||
|
|
@ -1,199 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
'''
|
|
||||||
Flask functions and routes for the dashboard page
|
|
||||||
'''
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import datetime
|
|
||||||
import time
|
|
||||||
import flask
|
|
||||||
|
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, stream_with_context
|
|
||||||
|
|
||||||
from Role_Manager import login_admin, login_read_only
|
|
||||||
from flask_login import login_required
|
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
|
||||||
##################################
|
|
||||||
# Import Project packages
|
|
||||||
##################################
|
|
||||||
from lib import ail_queues
|
|
||||||
from lib import ail_updates
|
|
||||||
|
|
||||||
from packages.Date import Date
|
|
||||||
|
|
||||||
# ============ VARIABLES ============
|
|
||||||
import Flask_config
|
|
||||||
|
|
||||||
app = Flask_config.app
|
|
||||||
config_loader = Flask_config.config_loader
|
|
||||||
baseUrl = Flask_config.baseUrl
|
|
||||||
r_serv_log = Flask_config.r_serv_log
|
|
||||||
|
|
||||||
max_dashboard_logs = Flask_config.max_dashboard_logs
|
|
||||||
|
|
||||||
dashboard = Blueprint('dashboard', __name__, template_folder='templates')
|
|
||||||
|
|
||||||
# ============ FUNCTIONS ============
|
|
||||||
def event_stream():
|
|
||||||
pubsub = r_serv_log.pubsub()
|
|
||||||
pubsub.psubscribe("Script" + '.*')
|
|
||||||
try:
|
|
||||||
for msg in pubsub.listen():
|
|
||||||
mtype = msg['type']
|
|
||||||
pattern = msg['pattern']
|
|
||||||
channel = msg['channel']
|
|
||||||
data = msg['data']
|
|
||||||
|
|
||||||
msg = {'channel': channel, 'type': mtype, 'pattern': pattern, 'data': data}
|
|
||||||
|
|
||||||
level = (msg['channel']).split('.')[1]
|
|
||||||
if msg['type'] == 'pmessage' and level != "DEBUG":
|
|
||||||
yield 'data: %s\n\n' % json.dumps(msg)
|
|
||||||
except GeneratorExit:
|
|
||||||
print("Generator Exited")
|
|
||||||
pubsub.unsubscribe()
|
|
||||||
|
|
||||||
def event_stream_dashboard():
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
data = {'queues': get_queues()}
|
|
||||||
yield f'data: {json.dumps(data)}\n\n'
|
|
||||||
time.sleep(1)
|
|
||||||
except GeneratorExit:
|
|
||||||
print("Generator dashboard Exited")
|
|
||||||
pass
|
|
||||||
|
|
||||||
def get_queues():
|
|
||||||
# We may want to put the llen in a pipeline to do only one query.
|
|
||||||
return ail_queues.get_modules_queues_stats()
|
|
||||||
|
|
||||||
def get_date_range(date_from, num_day):
|
|
||||||
date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2))
|
|
||||||
date_list = []
|
|
||||||
|
|
||||||
for i in range(0, num_day+1):
|
|
||||||
new_date = date.substract_day(i)
|
|
||||||
date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}')
|
|
||||||
|
|
||||||
return date_list
|
|
||||||
|
|
||||||
def dashboard_alert(log):
|
|
||||||
# check if we need to display this log
|
|
||||||
if len(log) > 50:
|
|
||||||
date = log[1:5]+log[6:8]+log[9:11]
|
|
||||||
utc_str = log[1:20]
|
|
||||||
log = log[46:].split(';')
|
|
||||||
if len(log) == 6:
|
|
||||||
date_time = datetime_from_utc_to_local(utc_str)
|
|
||||||
path = url_for('investigations_b.get_object_gid', gid=log[5])
|
|
||||||
|
|
||||||
res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
|
|
||||||
'paste': log[3], 'message': log[4], 'path': path}
|
|
||||||
return res
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def datetime_from_utc_to_local(utc_str):
|
|
||||||
utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S')
|
|
||||||
now_timestamp = time.time()
|
|
||||||
offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
|
|
||||||
local_time_str = (utc_datetime + offset).strftime('%H:%M:%S')
|
|
||||||
return local_time_str
|
|
||||||
|
|
||||||
# ============ ROUTES ============
|
|
||||||
|
|
||||||
@dashboard.route("/_logs")
|
|
||||||
@login_required
|
|
||||||
@login_read_only
|
|
||||||
def logs():
|
|
||||||
return flask.Response(event_stream(), mimetype="text/event-stream")
|
|
||||||
|
|
||||||
@dashboard.route("/_dashboard")
|
|
||||||
@login_required
|
|
||||||
@login_read_only
|
|
||||||
def _dashboard():
|
|
||||||
return flask.Response(stream_with_context(event_stream_dashboard()), content_type="text/event-stream")
|
|
||||||
|
|
||||||
@dashboard.route("/_get_last_logs_json")
|
|
||||||
@login_required
|
|
||||||
@login_read_only
|
|
||||||
def get_last_logs_json():
|
|
||||||
date = datetime.datetime.now().strftime("%Y%m%d")
|
|
||||||
|
|
||||||
max_day_search = 6
|
|
||||||
day_search = 0
|
|
||||||
warning_found = 0
|
|
||||||
warning_to_found = max_dashboard_logs
|
|
||||||
|
|
||||||
last_logs = []
|
|
||||||
|
|
||||||
date_range = get_date_range(date, max_day_search)
|
|
||||||
while max_day_search != day_search and warning_found != warning_to_found:
|
|
||||||
|
|
||||||
filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log'
|
|
||||||
filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(filename_log, 'r') as f:
|
|
||||||
lines = f.read().splitlines()
|
|
||||||
curr_index = -1
|
|
||||||
while warning_found != warning_to_found:
|
|
||||||
try:
|
|
||||||
# get lasts warning logs
|
|
||||||
log_warn = dashboard_alert(lines[curr_index])
|
|
||||||
if log_warn != False:
|
|
||||||
last_logs.append(log_warn)
|
|
||||||
warning_found = warning_found + 1
|
|
||||||
curr_index = curr_index - 1
|
|
||||||
|
|
||||||
except IndexError:
|
|
||||||
# check previous warning log file
|
|
||||||
day_search = day_search + 1
|
|
||||||
break
|
|
||||||
except FileNotFoundError:
|
|
||||||
# check previous warning log file
|
|
||||||
day_search = day_search + 1
|
|
||||||
|
|
||||||
return jsonify(list(reversed(last_logs)))
|
|
||||||
|
|
||||||
|
|
||||||
@dashboard.route("/_stuff", methods=['GET'])
|
|
||||||
@login_required
|
|
||||||
@login_read_only
|
|
||||||
def stuff():
|
|
||||||
return jsonify(row1=get_queues())
|
|
||||||
|
|
||||||
|
|
||||||
# TODO: ADD UPDATE NOTE BY USER
|
|
||||||
@dashboard.route("/")
|
|
||||||
@login_required
|
|
||||||
@login_read_only
|
|
||||||
def index():
|
|
||||||
default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
|
|
||||||
threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")
|
|
||||||
log_select = {10, 25, 50, 100}
|
|
||||||
log_select.add(max_dashboard_logs)
|
|
||||||
log_select = list(log_select)
|
|
||||||
log_select.sort()
|
|
||||||
|
|
||||||
# Check if update in progress
|
|
||||||
background_update = False
|
|
||||||
update_message = ''
|
|
||||||
if ail_updates.is_update_background_running():
|
|
||||||
background_update = True
|
|
||||||
update_message = ail_updates.AILBackgroundUpdate(ail_updates.get_update_background_version()).get_message()
|
|
||||||
|
|
||||||
return render_template("index.html", default_minute = default_minute,
|
|
||||||
threshold_stucked_module=threshold_stucked_module,
|
|
||||||
log_select=log_select, selected=max_dashboard_logs,
|
|
||||||
background_update=background_update, update_message=update_message)
|
|
||||||
|
|
||||||
|
|
||||||
# ========= REGISTRATION =========
|
|
||||||
app.register_blueprint(dashboard, url_prefix=baseUrl)
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<title>WordsTrend</title>
|
|
||||||
|
|
||||||
<script type="text/javascript">
|
|
||||||
$(document).ready(function () {
|
|
||||||
var data = [];
|
|
||||||
var t = new Date();
|
|
||||||
|
|
||||||
for (var i = 10; i >= 0; i--) {
|
|
||||||
var x = new Date(t.getTime() - i * 1000);
|
|
||||||
data.push([x, 0]);
|
|
||||||
}
|
|
||||||
|
|
||||||
var g = new Dygraph(document.getElementById("Graph"), data,
|
|
||||||
{
|
|
||||||
drawPoints: true,
|
|
||||||
showRoller: true,
|
|
||||||
//valueRange: [0.0, 1.2],
|
|
||||||
legend: 'always',
|
|
||||||
title: 'Pastes Queues Monitoring',
|
|
||||||
ylabel: 'Numbers of pastes waiting',
|
|
||||||
xlabel: 'Time',
|
|
||||||
});
|
|
||||||
// It sucks that these things aren't objects, and we need to store state in window.
|
|
||||||
window.intervalId = setInterval(function() {
|
|
||||||
var x = new Date(); // current time
|
|
||||||
var y = 6;
|
|
||||||
data.push([x, y]);
|
|
||||||
g.updateOptions( { 'file': data } );
|
|
||||||
}, 1000);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<div id="Graph" style="width:100%; height:100%;"></div>
|
|
||||||
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,340 +0,0 @@
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
|
|
||||||
<head>
|
|
||||||
<title>Analysis Information Leak framework Dashboard</title>
|
|
||||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
|
||||||
|
|
||||||
<!-- Core CSS -->
|
|
||||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
|
||||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
|
||||||
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
|
|
||||||
|
|
||||||
<!-- JS -->
|
|
||||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
|
||||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
|
||||||
|
|
||||||
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
|
|
||||||
data-urllog="{{ url_for('dashboard.logs') }}">
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style>
|
|
||||||
.tableQueue tbody tr td,
|
|
||||||
.tableQueue tbody tr th,
|
|
||||||
.tableQueue thead tr td,
|
|
||||||
.tableQueue thead tr th{
|
|
||||||
padding: 1px;
|
|
||||||
}
|
|
||||||
tr.table-disabled {
|
|
||||||
background-color: #f2f2f2;
|
|
||||||
}
|
|
||||||
tr.table-log-warning {
|
|
||||||
background-color: #ccfae3;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
window.default_minute = {{ default_minute }};
|
|
||||||
</script>
|
|
||||||
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
|
|
||||||
|
|
||||||
{% include 'nav_bar.html' %}
|
|
||||||
|
|
||||||
<div class="container-fluid">
|
|
||||||
<div class="row">
|
|
||||||
|
|
||||||
{% include 'dashboard/menu_sidebar.html' %}
|
|
||||||
|
|
||||||
<div class="col-12 col-lg-10" id="core_content">
|
|
||||||
|
|
||||||
{%if background_update%}
|
|
||||||
<div class="alert alert-warning alert-dismissible fade show my-2" role="alert">
|
|
||||||
<strong>Warning!</strong> An Update is running on the background. Some informations can be <strong>missing from the UI.</strong>
|
|
||||||
<strong>Updating: {{update_message}}</strong>
|
|
||||||
(<a href="{{ url_for('settings_b.settings_page') }}">Check Update Status</a>)
|
|
||||||
<button type="button" class="close" data-dismiss="alert" aria-label="Close">
|
|
||||||
<span aria-hidden="true">×</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
{%endif%}
|
|
||||||
|
|
||||||
<!-- TODO: Add users messages -->
|
|
||||||
{# {% include 'dashboard/update_modal.html' %}#}
|
|
||||||
|
|
||||||
<div class="row my-2">
|
|
||||||
|
|
||||||
<div class="col-xl-6">
|
|
||||||
<div class="card">
|
|
||||||
<div class="card-header">
|
|
||||||
<i class="far fa-chart-bar"></i> Feeder(s) Monitor:
|
|
||||||
</div>
|
|
||||||
<div class="card-body" id="panelbody"> {#style="height:420px;"#}
|
|
||||||
<strong>Processed items</strong>
|
|
||||||
<div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="col-xl-6">
|
|
||||||
{# <div class="card">#}
|
|
||||||
{# <div class="card-header">#}
|
|
||||||
{# <i class="far fa-chart-bar"></i> Queues Monitor#}
|
|
||||||
{# </div>#}
|
|
||||||
{# <div class="card-body">#}
|
|
||||||
{# <div class="" id="Graph" style="height:195px; width:88%;"></div>#}
|
|
||||||
{# <div style="height:10px;"></div>#}
|
|
||||||
{# <div class="" id="Graph2" style="height:195px; width:88%;"></div>#}
|
|
||||||
{# </div>#}
|
|
||||||
{##}
|
|
||||||
{# </div>#}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
<div class="card">
|
|
||||||
<div class="card-header">
|
|
||||||
<i class="fas fa-clipboard-list"></i> Logs
|
|
||||||
<div class="float-right">
|
|
||||||
<label style="padding-bottom:2px;">
|
|
||||||
<select class="form-control input-sm" id="log_select">
|
|
||||||
{% for log_selection in log_select %}
|
|
||||||
{% if log_selection == selected %}
|
|
||||||
<option value="{{ log_selection }}" selected>{{ log_selection }}</option>
|
|
||||||
{% else %}
|
|
||||||
<option value="{{ log_selection }}">{{ log_selection }}</option>
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
</select>
|
|
||||||
</label>
|
|
||||||
<input id="checkbox_log_info" type="checkbox" value="INFO"> INFO
|
|
||||||
<input id="checkbox_log_warning" type="checkbox" value="WARNING" checked="true"> WARNING
|
|
||||||
<input id="checkbox_log_critical" type="checkbox" value="CRITICAL" checked="true"> CRITICAL
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="card-body">
|
|
||||||
<table class="table table-hover table-sm" id="table_log">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Time</th>
|
|
||||||
<th>Channel</th>
|
|
||||||
<th>Level</th>
|
|
||||||
<th>Script Name</th>
|
|
||||||
<th>Source</th>
|
|
||||||
<th>Date</th>
|
|
||||||
<th>Paste name</th>
|
|
||||||
<th>Message</th>
|
|
||||||
<th>Actions</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody id="tab_body">
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</body>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<script>
|
|
||||||
var url_showSavedPath = "{{ url_for('investigations_b.get_object_gid') }}";
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
$("#page-Dashboard").addClass("active");
|
|
||||||
|
|
||||||
var tableBody = document.getElementById('tab_body')
|
|
||||||
$.getJSON("{{ url_for('dashboard.get_last_logs_json') }}", function(data) {
|
|
||||||
data.forEach(function (d) {
|
|
||||||
var tr = document.createElement('TR')
|
|
||||||
var time = document.createElement('TD')
|
|
||||||
var chan = document.createElement('TD')
|
|
||||||
var level = document.createElement('TD')
|
|
||||||
var scrpt = document.createElement('TD')
|
|
||||||
var src = document.createElement('TD')
|
|
||||||
var pdate = document.createElement('TD')
|
|
||||||
var nam = document.createElement('TD')
|
|
||||||
var msage = document.createElement('TD')
|
|
||||||
var inspect = document.createElement('TD')
|
|
||||||
|
|
||||||
// console.log(d)
|
|
||||||
|
|
||||||
tr.className = "table-log-warning";
|
|
||||||
time.appendChild(document.createTextNode(d.time))
|
|
||||||
chan.appendChild(document.createTextNode('Script'))
|
|
||||||
level.appendChild(document.createTextNode('WARNING'))
|
|
||||||
scrpt.appendChild(document.createTextNode(d.script))
|
|
||||||
src.appendChild(document.createTextNode(d.domain))
|
|
||||||
pdate.appendChild(document.createTextNode(d.date_paste))
|
|
||||||
nam.appendChild(document.createTextNode(d.paste))
|
|
||||||
|
|
||||||
var iconspan = document.createElement('SPAN');
|
|
||||||
var message = d.message.split(" ")
|
|
||||||
if (message[0] == "Detected"){
|
|
||||||
iconspan.className = "fas fa-eye";
|
|
||||||
}
|
|
||||||
else if (message[0] == "Checked"){
|
|
||||||
iconspan.className = "far fa-thumbs-up";
|
|
||||||
}
|
|
||||||
iconspan.innerHTML = " ";
|
|
||||||
msage.appendChild(iconspan);
|
|
||||||
msage.appendChild(document.createTextNode(message.join(" ")));
|
|
||||||
|
|
||||||
var action_icon_a = document.createElement("A");
|
|
||||||
action_icon_a.setAttribute("TARGET", "_blank");
|
|
||||||
action_icon_a.setAttribute("HREF", d.path);
|
|
||||||
var action_icon_span = document.createElement('SPAN');
|
|
||||||
action_icon_span.className = "fas fa-search-plus";
|
|
||||||
action_icon_a.appendChild(action_icon_span);
|
|
||||||
inspect.appendChild(action_icon_a)
|
|
||||||
inspect.setAttribute("style", "text-align:center;");
|
|
||||||
|
|
||||||
|
|
||||||
tr.appendChild(time)
|
|
||||||
tr.appendChild(chan);
|
|
||||||
tr.appendChild(level);
|
|
||||||
tr.appendChild(scrpt);
|
|
||||||
tr.appendChild(src);
|
|
||||||
tr.appendChild(pdate);
|
|
||||||
tr.appendChild(nam);
|
|
||||||
tr.appendChild(msage);
|
|
||||||
tr.appendChild(inspect);
|
|
||||||
|
|
||||||
tableBody.appendChild(tr);
|
|
||||||
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
function toggle_sidebar(){
|
|
||||||
if($('#nav_menu').is(':visible')){
|
|
||||||
$('#nav_menu').hide();
|
|
||||||
$('#side-bard-dashboard-content').hide();
|
|
||||||
$('#side_menu').removeClass('border-right')
|
|
||||||
$('#side_menu').removeClass('col-lg-2')
|
|
||||||
$('#core_content').removeClass('col-lg-10')
|
|
||||||
}else{
|
|
||||||
$('#nav_menu').show();
|
|
||||||
$('#side-bard-dashboard-content').show();
|
|
||||||
$('#side_menu').addClass('border-right')
|
|
||||||
$('#side_menu').addClass('col-lg-2')
|
|
||||||
$('#core_content').addClass('col-lg-10')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
|
|
||||||
|
|
||||||
<script>
|
|
||||||
|
|
||||||
function create_queue_table(data) {
|
|
||||||
document.getElementById("queueing").innerHTML = "";
|
|
||||||
let Tablediv = document.getElementById("queueing")
|
|
||||||
let table = document.createElement('TABLE')
|
|
||||||
table.className = "table table-bordered table-hover tableQueue";
|
|
||||||
let tableHead = document.createElement('THEAD')
|
|
||||||
let tableBody = document.createElement('TBODY')
|
|
||||||
|
|
||||||
table.appendChild(tableHead);
|
|
||||||
table.appendChild(tableBody);
|
|
||||||
let heading = new Array();
|
|
||||||
heading[0] = "Queue Name.PID"
|
|
||||||
heading[1] = "Amount"
|
|
||||||
let tr = document.createElement('TR');
|
|
||||||
tableHead.appendChild(tr);
|
|
||||||
|
|
||||||
for (i = 0; i < heading.length; i++) {
|
|
||||||
let th = document.createElement('TH')
|
|
||||||
if (heading[i] === "Amount") {
|
|
||||||
th.width = '50';
|
|
||||||
} else {
|
|
||||||
th.width = '100';
|
|
||||||
}
|
|
||||||
th.appendChild(document.createTextNode(heading[i]));
|
|
||||||
tr.appendChild(th);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((data).length === 0) {
|
|
||||||
let tr = document.createElement('TR');
|
|
||||||
let td = document.createElement('TD');
|
|
||||||
let td2 = document.createElement('TD');
|
|
||||||
td.appendChild(document.createTextNode("No running queues"));
|
|
||||||
td2.appendChild(document.createTextNode("Or no feed"));
|
|
||||||
td.className += " table-danger";
|
|
||||||
td2.className += " table-danger";
|
|
||||||
tr.appendChild(td);
|
|
||||||
tr.appendChild(td2);
|
|
||||||
tableBody.appendChild(tr);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
for(i = 0; i < (data).length;i++){
|
|
||||||
let tr = document.createElement('TR')
|
|
||||||
for(j = 0; j < 2; j++){
|
|
||||||
let td = document.createElement('TD')
|
|
||||||
let moduleNum = j == 0 ? "." + data[i][3] : "";
|
|
||||||
td.appendChild(document.createTextNode(data[i][j] + moduleNum));
|
|
||||||
tr.appendChild(td)
|
|
||||||
}
|
|
||||||
// Used to decide the color of the row
|
|
||||||
// We have data[][j] with:
|
|
||||||
// - j=0: ModuleName
|
|
||||||
// - j=1: queueLength
|
|
||||||
// - j=2: LastProcessedPasteTime
|
|
||||||
// - j=3: Number of the module belonging in the same category
|
|
||||||
if (data[i][3]==="Not Launched")
|
|
||||||
tr.className += " bg-danger text-white";
|
|
||||||
else if (parseInt(data[i][2]) > window.threshold_stucked_module && parseInt(data[i][1]) > 2)
|
|
||||||
tr.className += " table-danger";
|
|
||||||
else if (parseInt(data[i][1]) === 0)
|
|
||||||
tr.className += " table-disabled";
|
|
||||||
else
|
|
||||||
tr.className += " table-success";
|
|
||||||
tableBody.appendChild(tr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Tablediv.appendChild(table);
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
|
||||||
//
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
var event_queues
|
|
||||||
|
|
||||||
$(document).ready(function() {
|
|
||||||
// Hide Browser Error: connection interrupted while the page was loading
|
|
||||||
$(window).on("beforeunload", function() {
|
|
||||||
event_queues.close();
|
|
||||||
source.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
event_queues = new EventSource("{{ url_for('dashboard._dashboard') }}");
|
|
||||||
event_queues.onmessage = function(event) {
|
|
||||||
let data = JSON.parse(event.data);
|
|
||||||
create_queue_table(data['queues'])
|
|
||||||
};
|
|
||||||
event_queues.onerror = function(event) {
|
|
||||||
console.error('EventSource failed:', event);
|
|
||||||
event_queues.close();
|
|
||||||
};
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
</script>
|
|
||||||
|
|
||||||
</html>
|
|
||||||
434
var/www/templates/dashboard/dashboard.html
Normal file
434
var/www/templates/dashboard/dashboard.html
Normal file
|
|
@ -0,0 +1,434 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<title>Objects - AIL</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/echarts.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.popover {
|
||||||
|
max-width: none;
|
||||||
|
width: 500px;
|
||||||
|
}
|
||||||
|
.icon-button {
|
||||||
|
position: relative;
|
||||||
|
display: inline-block;
|
||||||
|
border: none;
|
||||||
|
background: none;
|
||||||
|
{#padding: 0;#}
|
||||||
|
margin: 0;
|
||||||
|
cursor: pointer;
|
||||||
|
outline: inherit;
|
||||||
|
transition: color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;
|
||||||
|
}
|
||||||
|
|
||||||
|
.icon-wrapper {
|
||||||
|
position: relative;
|
||||||
|
display: inline-block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notification-badge {
|
||||||
|
position: absolute;
|
||||||
|
top: -10px;
|
||||||
|
right: -10px;
|
||||||
|
font-size: 14px;
|
||||||
|
background: forestgreen;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
<div id="feeders_dashboard" style="width: 100%; height:600px;"></div>
|
||||||
|
|
||||||
|
<div class="card mt-1 mb-4">
|
||||||
|
<div class="card-header text-white bg-dark">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-1">
|
||||||
|
<i class="fas fa-user-secret fa-2x"></i>
|
||||||
|
</div>
|
||||||
|
<div class="col-5">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-6">
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&up=True" class="badge badge-success" id="stat_crawler_onion_domain_up">{{ crawlers_stats['onion']['up'] }}</a> Up
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&down=True" class="badge badge-danger" id="stat_crawler_onion_domain_down">{{ crawlers_stats['onion']['down'] }}</a> Down
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=onion&up=True&down=True" class="badge badge-success" id="stat_crawler_onion_total">{{ crawlers_stats['onion']['crawled'] }}</a> Crawled
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span class="badge badge-warning" id="stat_crawler_onion_queue">{{ crawlers_stats['onion']['queue'] }}</span> Queue
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-1">
|
||||||
|
<i class="fab fa-html5 fa-2x"></i>
|
||||||
|
</div>
|
||||||
|
<div class="col-5">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-6">
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&up=True" class="badge badge-success" id="stat_crawler_regular_domain_up">{{ crawlers_stats['web']['up'] }}</a> UP
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&down=True" class="badge badge-danger" id="stat_crawler_regular_domain_down">{{ crawlers_stats['web']['down'] }}</a> DOWN
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<div>
|
||||||
|
<a href="{{ url_for('crawler_splash.domains_search_today') }}?type=web&up=True&down=True" class="badge badge-success" id="stat_crawler_regular_total">{{ crawlers_stats['web']['crawled'] }}</a> Crawled
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span class="badge badge-warning" id="stat_crawler_regular_queue">{{ crawlers_stats['web']['queue'] }}</span> Queue
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-12 col-xl-6 mt-4">
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% for obj_type in nb_objects %}
|
||||||
|
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||||
|
{% with type=obj_type, name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %}
|
||||||
|
{% include 'objects/block_obj_button.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th><i class="fa-solid fa-crosshairs"></i></th>
|
||||||
|
<th>tracker</th>
|
||||||
|
<th class="text-center"><i class="fas fa-cube"></i></th>
|
||||||
|
<th>Time</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="tracked_objs">
|
||||||
|
{% for tracker in trackers %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ tracker['type'] }}</td>
|
||||||
|
<td style="word-break: break-all;">
|
||||||
|
<a href="{{ url_for('hunters.show_tracker') }}?uuid={{ tracker['uuid'] }}">{{ tracker['tracked'] }}</a>
|
||||||
|
</td>
|
||||||
|
<td class="text-center">
|
||||||
|
<a href="{{ tracker['obj']['link'] }}" onmouseenter="show_obj_tooltip(this, '{{ tracker['obj']['gid'] }}')" onmouseleave="hide_obj_tooltip(this)">
|
||||||
|
<svg height="26" width="26" style="pointer-events:none;">
|
||||||
|
<g class="nodes">
|
||||||
|
<circle cx="13" cy="13" r="13" fill="{{ tracker['obj']['icon']['color'] }}"></circle>
|
||||||
|
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ tracker['obj']['icon']['style'] }}" font-size="16px">{{ tracker['obj']['icon']['icon'] }}</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>{{ tracker['timestamp'] }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th class="text-center"><i class="fas fa-cube"></i></th>
|
||||||
|
<th>ID</th>
|
||||||
|
<th><i class="fas fa-tag"></i> Tags</th>
|
||||||
|
<th>Time</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="t_tags_objs">
|
||||||
|
{% for t_obj in tagged_objs %}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<a href="{{ t_obj['link'] }}" onmouseenter="show_obj_tooltip(this, '{{ t_obj['gid'] }}')" onmouseleave="hide_obj_tooltip(this)">
|
||||||
|
<svg height="26" width="26">
|
||||||
|
<g class="nodes">
|
||||||
|
<circle cx="13" cy="13" r="13" fill="{{ t_obj['icon']['color'] }}"></circle>
|
||||||
|
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ t_obj['icon']['style'] }}" font-size="16px">{{ t_obj['icon']['icon'] }}</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td style="word-break: break-all;">
|
||||||
|
<a href="{{ t_obj['link'] }}">{{ t_obj['id'] }}</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for tag in t_obj['tags'] %}
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
<td>{{ t_obj['date_tag'] }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var chart = {};
|
||||||
|
$(document).ready(function(){
|
||||||
|
$("#page-Decoded").addClass("active");
|
||||||
|
$("#nav_dashboard_{{obj_type}}").addClass("active");
|
||||||
|
});
|
||||||
|
|
||||||
|
var btn_selector = {
|
||||||
|
{% for obj_type in nb_objects %}
|
||||||
|
'{{ obj_type }}': $("#obj_btn_{{ obj_type }}"),
|
||||||
|
{% endfor %}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var tags_colors = ['primary', 'success', 'danger', 'warning', 'info'];
|
||||||
|
var url_obj_description = "{{ url_for('correlation.get_description') }}?object_id=";
|
||||||
|
|
||||||
|
// Init Dashboard
|
||||||
|
var feederChart = echarts.init(document.getElementById('feeders_dashboard'));
|
||||||
|
window.addEventListener('resize', function() {
|
||||||
|
feederChart.resize();
|
||||||
|
});
|
||||||
|
var optionFeeder;
|
||||||
|
var maxDataLength = 21;
|
||||||
|
|
||||||
|
var feeders_names = [{% for feeder in feeders_dashboard['data'] %}'{{ feeder }}', {% endfor %}]
|
||||||
|
var feeders_xaxis = [{% for date in feeders_dashboard['dates'] %}'{{ date }}', {% endfor %}]
|
||||||
|
var feeders_data = {
|
||||||
|
{% for feeder in feeders_dashboard['data'] %}
|
||||||
|
{{ feeder }}: {{ feeders_dashboard['data'][feeder] }},
|
||||||
|
{% endfor %}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
optionFeeder = {
|
||||||
|
title: {
|
||||||
|
text: 'Feeders'
|
||||||
|
},
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
axisPointer: {
|
||||||
|
type: 'cross',
|
||||||
|
label: {
|
||||||
|
backgroundColor: '#6a7985'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
legend: {
|
||||||
|
data: feeders_names
|
||||||
|
},
|
||||||
|
grid: {
|
||||||
|
left: '3%',
|
||||||
|
right: '4%',
|
||||||
|
bottom: '3%',
|
||||||
|
containLabel: true
|
||||||
|
},
|
||||||
|
xAxis: [
|
||||||
|
{
|
||||||
|
type: 'category',
|
||||||
|
boundaryGap: false,
|
||||||
|
data: feeders_xaxis
|
||||||
|
}
|
||||||
|
],
|
||||||
|
yAxis: [
|
||||||
|
{
|
||||||
|
type: 'value'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
series: [
|
||||||
|
{% for feeder in feeders_dashboard['data'] %}
|
||||||
|
{
|
||||||
|
name: '{{ feeder }}',
|
||||||
|
type: 'line',
|
||||||
|
stack: 'Total',
|
||||||
|
areaStyle: {},
|
||||||
|
emphasis: {
|
||||||
|
focus: 'series'
|
||||||
|
},
|
||||||
|
data: feeders_data['{{ feeder }}']
|
||||||
|
},
|
||||||
|
{% endfor %}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
optionFeeder && feederChart.setOption(optionFeeder);
|
||||||
|
|
||||||
|
{# TODO UPDATE NEW FEEDER NAME#}
|
||||||
|
function updateFeederChart(data) {
|
||||||
|
let new_date = data['date']
|
||||||
|
feeders_xaxis.push(new_date)
|
||||||
|
feeders_xaxis.shift()
|
||||||
|
|
||||||
|
for (const f_name in data['data']) {
|
||||||
|
if (f_name in feeders_data) {
|
||||||
|
feeders_data[f_name].push(data['data'][f_name])
|
||||||
|
feeders_data[f_name].shift()
|
||||||
|
} else {
|
||||||
|
let fdata = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, data['data'][f_name]]
|
||||||
|
feeders_names.push(f_name)
|
||||||
|
feeders_data[f_name] = fdata
|
||||||
|
// add new feeder
|
||||||
|
optionFeeder.legend.data.push(f_name)
|
||||||
|
optionFeeder.series.push({name: f_name, type: 'line', stack: 'Total', areaStyle: {}, emphasis: {focus: 'series'}, data: feeders_data[f_name]})
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
feederChart.setOption(optionFeeder)
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateNbObjects(data) {
|
||||||
|
for (const obj_type in data) {
|
||||||
|
btn_selector[obj_type].text(data[obj_type]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateCrawlerStats(data) {
|
||||||
|
$("#stat_crawler_onion_domain_up").text(data['onion']['up']);
|
||||||
|
$("#stat_crawler_onion_domain_down").text(data['onion']['down']);
|
||||||
|
$("#stat_crawler_onion_total").text(data['onion']['crawled']);
|
||||||
|
$("#stat_crawler_onion_queue").text(data['onion']['queue']);
|
||||||
|
$("#stat_crawler_regular_domain_up").text(data['web']['up']);
|
||||||
|
$("#stat_crawler_regular_domain_down").text(data['web']['down']);
|
||||||
|
$("#stat_crawler_regular_total").text(data['web']['crawled']);
|
||||||
|
$("#stat_crawler_regular_queue").text(data['web']['queue']);
|
||||||
|
}
|
||||||
|
|
||||||
|
///////
|
||||||
|
|
||||||
|
function create_obj_svg(container, obj_gid, url, color, fa_style, icon) {
|
||||||
|
var svg_obj = '<svg height="26" width="26"> <g class="nodes"> <circle cx="13" cy="13" r="13" fill="' + color + '"></circle> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="' + fa_style + '" font-size="16px">' + icon + '</text> </g> </svg>';
|
||||||
|
var obj_link = $('<a>');
|
||||||
|
obj_link.on('mouseenter', function () {
|
||||||
|
show_obj_tooltip(this, obj_gid);
|
||||||
|
});
|
||||||
|
obj_link.on('mouseleave', function () {
|
||||||
|
hide_obj_tooltip(this);
|
||||||
|
});
|
||||||
|
obj_link.attr('href', url);
|
||||||
|
obj_link.html(svg_obj);
|
||||||
|
container.append(obj_link);
|
||||||
|
return container
|
||||||
|
}
|
||||||
|
|
||||||
|
function create_tags_badges(container, tags) {
|
||||||
|
var tspan = '';
|
||||||
|
for (const i in tags) {
|
||||||
|
tspan = tspan + '<span class="badge badge-' + 'primary' + '">' + sanitize_text(tags[i]) + '</span>'
|
||||||
|
}
|
||||||
|
return tspan;
|
||||||
|
}
|
||||||
|
|
||||||
|
function update_tags_obj_dashboard(data) {
|
||||||
|
// delete table
|
||||||
|
var tbody = $("#t_tags_objs");
|
||||||
|
var ntd;
|
||||||
|
tbody.empty();
|
||||||
|
for (const elem in data) {
|
||||||
|
let obj = data[elem];
|
||||||
|
var row = $('<tr></tr>');
|
||||||
|
ntd = $('<td></td>');
|
||||||
|
row.append(create_obj_svg(ntd, obj['gid'], obj.link, obj['icon']['color'], obj['icon']['style'], obj['icon']['icon']))
|
||||||
|
row.append('<td style="word-break: break-all;"><a href="'+ obj['link'] + '">' + sanitize_text(obj['id']) + '</a></td>');
|
||||||
|
ntd = $('<td></td>');
|
||||||
|
row.append(create_tags_badges(ntd, obj['tags']));
|
||||||
|
row.append('<td>' + obj['date_tag'] + '</td>');
|
||||||
|
tbody.append(row);
|
||||||
|
|
||||||
|
|
||||||
|
{# {% for tag in t_obj['tags'] %}#}
|
||||||
|
{# <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>#}
|
||||||
|
{# {% endfor %}#}
|
||||||
|
{#</td>#}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function update_tracked_objs_dashboard(data) {
|
||||||
|
// delete table
|
||||||
|
var tbody = $("#tracked_objs");
|
||||||
|
var ntd;
|
||||||
|
tbody.empty();
|
||||||
|
for (const elem in data) {
|
||||||
|
let tracker = data[elem];
|
||||||
|
var row = $('<tr></tr>');
|
||||||
|
row.append('<td>' + tracker['type'] + '</td>');
|
||||||
|
row.append('<td style="word-break: break-all;"><a href="{{ url_for('hunters.show_tracker') }}?uuid=' + tracker['uuid'] + '">' + sanitize_text(tracker['tracked']) + '</a></td>');
|
||||||
|
ntd = $('<td class="text-center"></td>');
|
||||||
|
row.append(create_obj_svg(ntd, tracker['obj']['gid'], tracker['obj']['link'], tracker['obj']['icon']['color'], tracker['obj']['icon']['style'], tracker['obj']['icon']['icon']));
|
||||||
|
row.append('<td>' + tracker['timestamp'] + '</td>');
|
||||||
|
tbody.append(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WebSocket
|
||||||
|
var socket = new WebSocket("{{ url_for('ws_dashboard') }}");
|
||||||
|
socket.wsocket = function(event) {
|
||||||
|
console.log("WebSocket connection opened:", event);
|
||||||
|
};
|
||||||
|
socket.onmessage = function(event) {
|
||||||
|
$("[data-toggle='popover']").popover('destroy');
|
||||||
|
let data = JSON.parse(event.data);
|
||||||
|
updateFeederChart(data['feeders']);
|
||||||
|
updateNbObjects(data['objs']);
|
||||||
|
updateCrawlerStats(data['crawler']);
|
||||||
|
update_tracked_objs_dashboard(data['trackers']);
|
||||||
|
update_tags_obj_dashboard(data['tags']);
|
||||||
|
};
|
||||||
|
socket.onerror = function(error) {
|
||||||
|
console.error('WebSocket error:', error);
|
||||||
|
};
|
||||||
|
socket.onclose = function(event) {
|
||||||
|
console.log('WebSocket connection closed:', event);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
|
|
||||||
|
|
||||||
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
|
|
||||||
<i class="fas fa-align-left"></i>
|
|
||||||
<span>Toggle Sidebar</span>
|
|
||||||
</button>
|
|
||||||
|
|
||||||
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
|
|
||||||
|
|
||||||
</nav>
|
|
||||||
|
|
||||||
<span id="side-bard-dashboard-content">
|
|
||||||
<div class="card">
|
|
||||||
<div class="card-header">
|
|
||||||
<i class="fas fa-tachometer-alt"></i> Total pastes since {{ default_minute }} min
|
|
||||||
</div>
|
|
||||||
<div class="card-body">
|
|
||||||
<div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="custom-control custom-switch ml-5 mt-2">
|
|
||||||
<input class="custom-control-input" id="button-toggle-queues" type="checkbox" checked>
|
|
||||||
<label class="custom-control-label" for="button-toggle-queues"><strong style="top: 3px; position: relative;">Display queues</strong></label>
|
|
||||||
</div>
|
|
||||||
<hr>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<div>
|
|
||||||
<table id="queue-color-legend" class="table table-borderless table-sm">
|
|
||||||
<tbody>
|
|
||||||
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #e8e8e8;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
|
|
||||||
<tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
|
|
||||||
<tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
|
|
||||||
</div>
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
<div class="navbar-header">
|
|
||||||
<div class="nav navbar-nav">
|
|
||||||
<!--dashboard-->
|
|
||||||
<!--PasteSubmit-->
|
|
||||||
<!--Tags-->
|
|
||||||
<!--terms-->
|
|
||||||
<!--browsepastes-->
|
|
||||||
<!--trendingcharts-->
|
|
||||||
<!--trendingmodules-->
|
|
||||||
<!--sentiment-->
|
|
||||||
<!--insert here-->
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
<li id='page-MODULENAME'><a href="{{ url_for('MODULENAME.MODULENAME_page') }}"><i class="glyphicon glyphicon-new-window"></i> MODULENAME </a></li>
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
rawSkeleton
|
|
||||||
Loading…
Reference in a new issue