diff --git a/var/www/modules/Flask_config.py b/var/www/Flask_config.py similarity index 100% rename from var/www/modules/Flask_config.py rename to var/www/Flask_config.py diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c502d873..2f34f857 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -14,11 +14,6 @@ from flask import Flask, render_template, jsonify, request, Request, Response, s from flask_login import LoginManager, current_user, login_user, logout_user, login_required from flask_sock import Sock -import importlib -from os.path import join - -sys.path.append('./modules/') - sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages @@ -37,6 +32,8 @@ import Flask_config # Import Blueprint from blueprints.root import root +from blueprints.dashboard import dashboard +from blueprints.ui_submit import PasteSubmit # TODO RENAME ME from blueprints.crawler_splash import crawler_splash from blueprints.correlation import correlation from blueprints.languages_ui import languages_ui @@ -48,7 +45,6 @@ from blueprints.hunters import hunters from blueprints.old_endpoints import old_endpoints from blueprints.ail_2_ail_sync import ail_2_ail_sync from blueprints.settings_b import settings_b -from blueprints.objects_objs import objects_objs from blueprints.objects_cve import objects_cve from blueprints.objects_decoded import objects_decoded from blueprints.objects_subtypes import objects_subtypes @@ -124,6 +120,8 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024 # ========= BLUEPRINT =========# app.register_blueprint(root, url_prefix=baseUrl) +app.register_blueprint(dashboard, url_prefix=baseUrl) +app.register_blueprint(PasteSubmit, url_prefix=baseUrl) app.register_blueprint(crawler_splash, url_prefix=baseUrl) app.register_blueprint(correlation, url_prefix=baseUrl) app.register_blueprint(languages_ui, url_prefix=baseUrl) @@ -136,7 +134,6 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl) app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl) app.register_blueprint(settings_b, url_prefix=baseUrl) app.register_blueprint(objects_cve, url_prefix=baseUrl) -app.register_blueprint(objects_objs, url_prefix=baseUrl) app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_subtypes, url_prefix=baseUrl) app.register_blueprint(objects_title, url_prefix=baseUrl) @@ -175,37 +172,6 @@ def load_user(session_id): return user return None -# ========= HEADER GENERATION ======== DEPRECATED - - -# Get headers items that should be ignored (not displayed) -toIgnoreModule = set() -try: - with open('templates/ignored_modules.txt', 'r') as f: - lines = f.read().splitlines() - for line in lines: - toIgnoreModule.add(line) - -except IOError: - pass - -# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################ -for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')): - sys.path.append(join(root)) - - # Ignore the module - curr_dir = root.split('/')[1] - if curr_dir in toIgnoreModule: - continue - - for name in files: - module_name = root.split('/')[-2] - if name.startswith('Flask_') and name.endswith('.py'): - if name == 'Flask_config.py': - continue - name = name.strip('.py') - importlib.import_module(name) - # ========= JINJA2 FUNCTIONS ======== def list_len(s): diff --git a/var/www/modules/Role_Manager.py b/var/www/Role_Manager.py similarity index 100% rename from var/www/modules/Role_Manager.py rename to var/www/Role_Manager.py diff --git a/var/www/blueprints/objects_objs.py b/var/www/blueprints/dashboard.py similarity index 74% rename from var/www/blueprints/objects_objs.py rename to var/www/blueprints/dashboard.py index 2b640e06..e66620d3 100644 --- a/var/www/blueprints/objects_objs.py +++ b/var/www/blueprints/dashboard.py @@ -7,9 +7,11 @@ import os import sys -import json +import time -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context +import json # TODO REMOVE ME + +from flask import render_template, Response, request, Blueprint from flask_login import login_required, current_user # Import Role_Manager @@ -21,21 +23,23 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_stats - +# TODO STATS +# from lib import ail_updates # ============ BLUEPRINT ============ -objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects')) - +dashboard = Blueprint('dashboard', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/dashboard')) # ============ VARIABLES ============ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] - # ============ FUNCTIONS ============ -@objects_objs.route("/objects", methods=['GET']) + +# ============= ROUTES ============== + +@dashboard.route("/", methods=['GET']) @login_required @login_read_only -def objects(): +def index(): user_org = current_user.get_org() user_id = current_user.get_user_id() nb_objects = ail_stats.get_nb_objs_dashboard() @@ -43,11 +47,6 @@ def objects(): crawlers_stats = ail_stats.get_crawlers_stats() trackers = ail_stats.get_tracked_objs_dashboard(user_org, user_id) tagged_objs = ail_stats.get_tagged_objs_dashboard() - return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard, + return render_template("dashboard.html", feeders_dashboard=feeders_dashboard, nb_objects=nb_objects, trackers=trackers, tagged_objs=tagged_objs, bootstrap_label=bootstrap_label, crawlers_stats=crawlers_stats) - - - -# ============= ROUTES ============== - diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/blueprints/ui_submit.py similarity index 98% rename from var/www/modules/PasteSubmit/Flask_PasteSubmit.py rename to var/www/blueprints/ui_submit.py index b478cdb3..3ce0cb88 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/blueprints/ui_submit.py @@ -50,7 +50,7 @@ text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000) file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000) allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS) -PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates') +PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/submit')) # ============ Validators ============ def limit_content_length(): @@ -296,6 +296,3 @@ def submit_status(): else: return 'INVALID UUID' - -# ========= REGISTRATION ========= -app.register_blueprint(PasteSubmit, url_prefix=baseUrl) diff --git a/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html b/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html deleted file mode 100644 index 4a6a3b77..00000000 --- a/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html +++ /dev/null @@ -1 +0,0 @@ -
  • Submit Paste
    • diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py deleted file mode 100644 index fd108fe6..00000000 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the dashboard page -''' -import json -import os -import sys -import datetime -import time -import flask - -from flask import Flask, render_template, jsonify, request, Blueprint, url_for, stream_with_context - -from Role_Manager import login_admin, login_read_only -from flask_login import login_required - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ail_queues -from lib import ail_updates - -from packages.Date import Date - -# ============ VARIABLES ============ -import Flask_config - -app = Flask_config.app -config_loader = Flask_config.config_loader -baseUrl = Flask_config.baseUrl -r_serv_log = Flask_config.r_serv_log - -max_dashboard_logs = Flask_config.max_dashboard_logs - -dashboard = Blueprint('dashboard', __name__, template_folder='templates') - -# ============ FUNCTIONS ============ -def event_stream(): - pubsub = r_serv_log.pubsub() - pubsub.psubscribe("Script" + '.*') - try: - for msg in pubsub.listen(): - mtype = msg['type'] - pattern = msg['pattern'] - channel = msg['channel'] - data = msg['data'] - - msg = {'channel': channel, 'type': mtype, 'pattern': pattern, 'data': data} - - level = (msg['channel']).split('.')[1] - if msg['type'] == 'pmessage' and level != "DEBUG": - yield 'data: %s\n\n' % json.dumps(msg) - except GeneratorExit: - print("Generator Exited") - pubsub.unsubscribe() - -def event_stream_dashboard(): - try: - while True: - data = {'queues': get_queues()} - yield f'data: {json.dumps(data)}\n\n' - time.sleep(1) - except GeneratorExit: - print("Generator dashboard Exited") - pass - -def get_queues(): - # We may want to put the llen in a pipeline to do only one query. - return ail_queues.get_modules_queues_stats() - -def get_date_range(date_from, num_day): - date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2)) - date_list = [] - - for i in range(0, num_day+1): - new_date = date.substract_day(i) - date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}') - - return date_list - -def dashboard_alert(log): - # check if we need to display this log - if len(log) > 50: - date = log[1:5]+log[6:8]+log[9:11] - utc_str = log[1:20] - log = log[46:].split(';') - if len(log) == 6: - date_time = datetime_from_utc_to_local(utc_str) - path = url_for('investigations_b.get_object_gid', gid=log[5]) - - res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], - 'paste': log[3], 'message': log[4], 'path': path} - return res - else: - return False - else: - return False - -def datetime_from_utc_to_local(utc_str): - utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S') - now_timestamp = time.time() - offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp) - local_time_str = (utc_datetime + offset).strftime('%H:%M:%S') - return local_time_str - -# ============ ROUTES ============ - -@dashboard.route("/_logs") -@login_required -@login_read_only -def logs(): - return flask.Response(event_stream(), mimetype="text/event-stream") - -@dashboard.route("/_dashboard") -@login_required -@login_read_only -def _dashboard(): - return flask.Response(stream_with_context(event_stream_dashboard()), content_type="text/event-stream") - -@dashboard.route("/_get_last_logs_json") -@login_required -@login_read_only -def get_last_logs_json(): - date = datetime.datetime.now().strftime("%Y%m%d") - - max_day_search = 6 - day_search = 0 - warning_found = 0 - warning_to_found = max_dashboard_logs - - last_logs = [] - - date_range = get_date_range(date, max_day_search) - while max_day_search != day_search and warning_found != warning_to_found: - - filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log' - filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log) - - try: - with open(filename_log, 'r') as f: - lines = f.read().splitlines() - curr_index = -1 - while warning_found != warning_to_found: - try: - # get lasts warning logs - log_warn = dashboard_alert(lines[curr_index]) - if log_warn != False: - last_logs.append(log_warn) - warning_found = warning_found + 1 - curr_index = curr_index - 1 - - except IndexError: - # check previous warning log file - day_search = day_search + 1 - break - except FileNotFoundError: - # check previous warning log file - day_search = day_search + 1 - - return jsonify(list(reversed(last_logs))) - - -@dashboard.route("/_stuff", methods=['GET']) -@login_required -@login_read_only -def stuff(): - return jsonify(row1=get_queues()) - - -# TODO: ADD UPDATE NOTE BY USER -@dashboard.route("/") -@login_required -@login_read_only -def index(): - default_minute = config_loader.get_config_str("Flask", "minute_processed_paste") - threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module") - log_select = {10, 25, 50, 100} - log_select.add(max_dashboard_logs) - log_select = list(log_select) - log_select.sort() - - # Check if update in progress - background_update = False - update_message = '' - if ail_updates.is_update_background_running(): - background_update = True - update_message = ail_updates.AILBackgroundUpdate(ail_updates.get_update_background_version()).get_message() - - return render_template("index.html", default_minute = default_minute, - threshold_stucked_module=threshold_stucked_module, - log_select=log_select, selected=max_dashboard_logs, - background_update=background_update, update_message=update_message) - - -# ========= REGISTRATION ========= -app.register_blueprint(dashboard, url_prefix=baseUrl) diff --git a/var/www/modules/dashboard/templates/Queue_live_Monitoring.html b/var/www/modules/dashboard/templates/Queue_live_Monitoring.html deleted file mode 100644 index dc278f23..00000000 --- a/var/www/modules/dashboard/templates/Queue_live_Monitoring.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - -WordsTrend - - - -
    - - - diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html deleted file mode 100644 index 10dc5a03..00000000 --- a/var/www/modules/dashboard/templates/index.html +++ /dev/null @@ -1,340 +0,0 @@ - - - - - Analysis Information Leak framework Dashboard - - - - - - - - - - - - - - - - - - - - - - - - -{% include 'nav_bar.html' %} - -
    -
    - - {% include 'dashboard/menu_sidebar.html' %} - -
    - - {%if background_update%} - - {%endif%} - - -{# {% include 'dashboard/update_modal.html' %}#} - -
    - -
    -
    -
    - Feeder(s) Monitor: -
    -
    {#style="height:420px;"#} - Processed items -
    -
    -
    -
    - -
    -{#
    #} -{#
    #} -{# Queues Monitor#} -{#
    #} -{#
    #} -{#
    #} -{#
    #} -{#
    #} -{#
    #} -{##} -{#
    #} -
    -
    - - -
    -
    - Logs -
    - - INFO - WARNING - CRITICAL - -
    -
    - -
    - - - - - - - - - - - - - - - - -
    TimeChannelLevelScript NameSourceDatePaste nameMessageActions
    -
    -
    - - -
    - - - -
    -
    - - - - - - - - - - - - - diff --git a/var/www/templates/dashboard/dashboard.html b/var/www/templates/dashboard/dashboard.html new file mode 100644 index 00000000..ae17b3db --- /dev/null +++ b/var/www/templates/dashboard/dashboard.html @@ -0,0 +1,434 @@ + + + + + Objects - AIL + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    + + +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    +
    +
    + + +
    +
    + +
    + {{ crawlers_stats['onion']['queue'] }} Queue +
    +
    +
    +
    +
    + +
    +
    +
    +
    + + +
    +
    + +
    + {{ crawlers_stats['web']['queue'] }} Queue +
    +
    +
    +
    +
    +
    +
    +
    + +
    + +
    + + {% for obj_type in nb_objects %} +
    + {% with type=obj_type, name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %} + {% include 'objects/block_obj_button.html' %} + {% endwith %} +
    + {% endfor %} + +
    + +
    + +
    + + +
    + +
    + + + + + + + + + + + {% for tracker in trackers %} + + + + + + + {% endfor %} + +
    trackerTime
    {{ tracker['type'] }} + {{ tracker['tracked'] }} + + + + + + {{ tracker['obj']['icon']['icon'] }} + + + + {{ tracker['timestamp'] }}
    +
    + +
    + + + + + + + + + + + {% for t_obj in tagged_objs %} + + + + + + + {% endfor %} + +
    ID TagsTime
    + + + + + {{ t_obj['icon']['icon'] }} + + + + + {{ t_obj['id'] }} + + {% for tag in t_obj['tags'] %} + {{ tag }} + {% endfor %} + {{ t_obj['date_tag'] }}
    +
    + +
    + + +
    + + + + + + + diff --git a/var/www/templates/dashboard/menu_sidebar.html b/var/www/templates/dashboard/menu_sidebar.html deleted file mode 100644 index b1abd954..00000000 --- a/var/www/templates/dashboard/menu_sidebar.html +++ /dev/null @@ -1,41 +0,0 @@ -
    - - - - - - -
    -
    - Total pastes since {{ default_minute }} min -
    -
    -
    -
    -
    - -
    - - -
    -
    - -
    -
    - - - - - - -
    		Idling queues
    		 Working queues
    		 Stuck queues
    -
    -
    -
    -     -
    diff --git a/var/www/templates/header_base.html b/var/www/templates/header_base.html deleted file mode 100644 index 9f2bff8e..00000000 --- a/var/www/templates/header_base.html +++ /dev/null @@ -1,13 +0,0 @@ -
    -
    - - - - - - - - - -
    -
    diff --git a/var/www/templates/header_base_template.html b/var/www/templates/header_base_template.html deleted file mode 100644 index ba2e43df..00000000 --- a/var/www/templates/header_base_template.html +++ /dev/null @@ -1 +0,0 @@ -
  • MODULENAME
    • diff --git a/var/www/templates/ignored_modules.txt b/var/www/templates/ignored_modules.txt deleted file mode 100644 index 742bc5fc..00000000 --- a/var/www/templates/ignored_modules.txt +++ /dev/null @@ -1 +0,0 @@ -rawSkeleton diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/templates/submit/submit_items.html similarity index 100% rename from var/www/modules/PasteSubmit/templates/submit_items.html rename to var/www/templates/submit/submit_items.html