From 721b07655925abff4943eb53a441c67f94969be7 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 6 Nov 2024 11:24:02 +0100 Subject: [PATCH] chg: [fontawesome] v6.6.0 migration --- .../dashboard/templates/header_dashboard.html | 1 - .../dashboard/templates/searchbox.html | 17 - var/www/modules/restApi/Flask_restApi.py | 528 ------------------ .../restApi/templates/api_default.html | 35 -- var/www/modules/search/Flask_search.py | 249 --------- var/www/modules/search/templates/search.html | 329 ----------- var/www/modules/sentiment/Flask_sentiment.py | 160 ------ .../sentiment/templates/header_sentiment.html | 7 - .../sentiment_analysis_plot_tool.html | 134 ----- .../sentiment_analysis_trending.html | 251 --------- .../trendingcharts/Flask_trendingcharts.py | 99 ---- .../trendingcharts/templates/Trending.html | 154 ----- .../trending_graphs/Domainstrending.html | 82 --- .../trending_graphs/Moduletrending.html | 96 ---- .../trending_graphs/Protocolstrending.html | 82 --- .../trending_graphs/Tldstrending.html | 84 --- .../trending_graphs/Wordstrending.html | 37 -- .../trendingmodules/Flask_trendingmodules.py | 143 ----- .../templates/Moduletrending.html | 73 --- .../templates/header_trendingmodules.html | 3 - var/www/templates/base_template.html | 42 -- var/www/templates/navbar.html | 19 - var/www/update_thirdparty.sh | 10 +- 23 files changed, 3 insertions(+), 2632 deletions(-) delete mode 100644 var/www/modules/dashboard/templates/header_dashboard.html delete mode 100644 var/www/modules/dashboard/templates/searchbox.html delete mode 100644 var/www/modules/restApi/Flask_restApi.py delete mode 100644 var/www/modules/restApi/templates/api_default.html delete mode 100644 var/www/modules/search/Flask_search.py delete mode 100644 var/www/modules/search/templates/search.html delete mode 100644 var/www/modules/sentiment/Flask_sentiment.py delete mode 100644 var/www/modules/sentiment/templates/header_sentiment.html delete mode 100644 var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html delete mode 100644 var/www/modules/sentiment/templates/sentiment_analysis_trending.html delete mode 100644 var/www/modules/trendingcharts/Flask_trendingcharts.py delete mode 100644 var/www/modules/trendingcharts/templates/Trending.html delete mode 100644 var/www/modules/trendingcharts/templates/trending_graphs/Domainstrending.html delete mode 100644 var/www/modules/trendingcharts/templates/trending_graphs/Moduletrending.html delete mode 100644 var/www/modules/trendingcharts/templates/trending_graphs/Protocolstrending.html delete mode 100644 var/www/modules/trendingcharts/templates/trending_graphs/Tldstrending.html delete mode 100644 var/www/modules/trendingcharts/templates/trending_graphs/Wordstrending.html delete mode 100644 var/www/modules/trendingmodules/Flask_trendingmodules.py delete mode 100644 var/www/modules/trendingmodules/templates/Moduletrending.html delete mode 100644 var/www/modules/trendingmodules/templates/header_trendingmodules.html delete mode 100644 var/www/templates/base_template.html delete mode 100644 var/www/templates/navbar.html diff --git a/var/www/modules/dashboard/templates/header_dashboard.html b/var/www/modules/dashboard/templates/header_dashboard.html deleted file mode 100644 index ea48e917..00000000 --- a/var/www/modules/dashboard/templates/header_dashboard.html +++ /dev/null @@ -1 +0,0 @@ -
  • Dashboard
    • diff --git a/var/www/modules/dashboard/templates/searchbox.html b/var/www/modules/dashboard/templates/searchbox.html deleted file mode 100644 index fd4cfb8e..00000000 --- a/var/www/modules/dashboard/templates/searchbox.html +++ /dev/null @@ -1,17 +0,0 @@ -
    - -
    - diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py deleted file mode 100644 index 92450579..00000000 --- a/var/www/modules/restApi/Flask_restApi.py +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -""" - Flask functions and routes for the rest api -""" - -# import os -# import re -# import sys -# import uuid -# import json - -# sys.path.append(os.environ['AIL_BIN']) -# ################################## -# # Import Project packages -# ################################## -# from lib.ConfigLoader import ConfigLoader -# from lib import Users -# from lib.objects import Items -# from lib import Tag -# -# from packages import Import_helper -# -# from importer.FeederImporter import api_add_json_feeder_to_queue -# -# -# from flask import jsonify, request, Blueprint, redirect, url_for, Response -# -# from functools import wraps - -# ============ VARIABLES ============ - - - -# ============ DECORATOR ============ - - - - - -# ============= ROUTES ============== - -# @restApi.route("/api", methods=['GET']) -# @login_required -# def api(): -# return 'api doc' - - -''' -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# POST -# -# { -# "id": item_id, mandatory -# "content": true, -# -# -# } -# -# response: { -# "id": "item_id", -# "tags": [], -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/item", methods=['POST']) -@token_required('read_only') -def get_item_id(): - data = request.get_json() - res = Items.api_get_item(data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# GET -# -# { -# "id": item_id, mandatory -# } -# -# response: { -# "id": "item_id", -# "date": "date", -# "tags": [], -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/item/tag", methods=['POST']) -@token_required('read_only') -def get_item_tag(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': False, 'tags': True} - res = Item.get_item(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# POST -# -# { -# "id": item_id, mandatory -# "tags": [tags to add], -# "galaxy": [galaxy to add], -# } -# -# response: { -# "id": "item_id", -# "tags": [tags added], -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/add/item/tag", methods=['POST']) -@token_required('analyst') -def add_item_tags(): - - data = request.get_json() - if not data: - return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 - - object_id = data.get('id', None) - tags = data.get('tags', []) - galaxy = data.get('galaxy', []) - - # res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item") - res = {'error': 'disabled endpoint'}, 500 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# DELETE -# -# { -# "id": item_id, mandatory -# "tags": [tags to delete], -# } -# -# response: { -# "id": "item_id", -# "tags": [tags deleted], -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/delete/item/tag", methods=['DELETE']) -@token_required('analyst') -def delete_item_tags(): - - data = request.get_json() - if not data: - return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 - - object_id = data.get('id', None) - tags = data.get('tags', []) - - res = Tag.api_delete_obj_tags(tags=tags, object_id=object_id, object_type="item") - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# GET -# -# { -# "id": item_id, mandatory -# } -# -# response: { -# "id": "item_id", -# "content": "item content" -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/item/content", methods=['POST']) -@token_required('read_only') -def get_item_content(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False} - res = Item.get_item(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - - -@restApi.route("api/v1/get/item/content/utf8/base64", methods=['POST']) -@token_required('read_only') -def get_item_content_encoded_text(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id} - res = Item.api_get_item_content_base64_utf8(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - - -@restApi.route("api/v1/get/items/sources", methods=['GET']) -@token_required('read_only') -def get_item_sources(): - res = Item.api_get_items_sources() - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] -''' - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -''' -@restApi.route("api/v1/get/tag/metadata", methods=['POST']) -@token_required('read_only') -def get_tag_metadata(): - data = request.get_json() - tag = data.get('tag', None) - if not Tag.is_tag_in_all_tag(tag): - return Response(json.dumps({'status': 'error', 'reason':'Tag not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404 - metadata = Tag.get_tag_metadata(tag) - return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200 - -@restApi.route("api/v1/get/tag/all", methods=['GET']) -@token_required('read_only') -def get_all_tags(): - res = {'tags': Tag.get_all_tags()} - return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 -''' - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO -# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # TODO -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO -''' -@restApi.route("api/v1/add/tracker", methods=['POST']) -@token_required('analyst') -def add_tracker_term(): - data = request.get_json() - user_token = get_auth_from_header() - user_id = Users.get_token_user(user_token) - res = Tracker.api_add_tracker(data, user_id) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/delete/tracker", methods=['DELETE']) -@token_required('analyst') -def delete_tracker_term(): - data = request.get_json() - user_token = get_auth_from_header() - user_id = Users.get_token_user(user_token) - res = Term.parse_tracked_term_to_delete(data, user_id) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/tracker/item", methods=['POST']) -@token_required('read_only') -def get_tracker_term_item(): - data = request.get_json() - user_token = get_auth_from_header() - user_id = Users.get_token_user(user_token) - res = Term.parse_get_tracker_term_item(data, user_id) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - - -@restApi.route("api/v1/get/tracker/yara/content", methods=['POST']) -@token_required('read_only') -def get_default_yara_rule_content(): - data = request.get_json() - rule_name = data.get('rule_name', None) - rule_name = escape(rule_name) - req_data = {'rule_name': rule_name} - res = Tracker.get_yara_rule_content_restapi(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/tracker/metadata", methods=['POST']) -@token_required('read_only') -def get_tracker_metadata_api(): - data = request.get_json() - tracker_uuid = data.get('tracker_uuid', None) - req_data = {'tracker_uuid': tracker_uuid} - - tracker_uuid = request_dict.get('tracker_uuid', None) - if not request_dict: - return {'status': 'error', 'reason': 'Malformed JSON'}, 400 - if not tracker_uuid: - return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 - if not is_valid_uuid_v4(tracker_uuid): - return {"status": "error", "reason": "Invalid Tracker UUID"}, 400 - if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): - return {'status': 'error', 'reason': 'Tracker not found'}, 404 - - res = Tracker.get_tracker_metadata_api(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1] - -''' - -''' - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) -@token_required('read_only') -def get_cryptocurrency_bitcoin_metadata(): - data = request.get_json() - crypto_address = data.get('bitcoin', None) - req_data = {'bitcoin': crypto_address, 'metadata': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) -@token_required('read_only') -def get_cryptocurrency_bitcoin_item(): - data = request.get_json() - bitcoin_address = data.get('bitcoin', None) - req_data = {'bitcoin': bitcoin_address, 'items': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) -@token_required('read_only') -def get_pgp_key_metadata(): - data = request.get_json() - pgp_field = data.get('key', None) - req_data = {'key': pgp_field, 'metadata': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) -@token_required('read_only') -def get_pgp_mail_metadata(): - data = request.get_json() - pgp_field = data.get('mail', None) - req_data = {'mail': pgp_field, 'metadata': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) -@token_required('read_only') -def get_pgp_name_metadata(): - data = request.get_json() - pgp_field = data.get('name', None) - req_data = {'name': pgp_field, 'metadata': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/pgp/key/item", methods=['POST']) -@token_required('read_only') -def get_pgp_key_item(): - data = request.get_json() - pgp_field = data.get('key', None) - req_data = {'key': pgp_field, 'items': True} - res = 0 - raise Exception('TO MIGRATE') - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) -@token_required('read_only') -def get_pgp_mail_item(): - data = request.get_json() - pgp_mail = data.get('mail', None) - req_data = {'mail': pgp_mail, 'items': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/pgp/name/item", methods=['POST']) -@token_required('read_only') -def get_pgp_name_item(): - data = request.get_json() - pgp_name = data.get('name', None) - req_data = {'name': pgp_name, 'items': True} - raise Exception('TO MIGRATE') - res = 0 - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - - - -@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) -@token_required('analyst') -def get_item_cryptocurrency_bitcoin(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}} - res = Item.get_item(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/item/pgp/mail", methods=['POST']) -@token_required('analyst') -def get_item_cryptocurrency_bitcoin(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}} - res = Item.get_item(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/item/pgp/name", methods=['POST']) -@token_required('analyst') -def get_item_cryptocurrency_bitcoin(): - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}} - res = Item.get_item(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] -''' - - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # DOMAIN # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -''' -@restApi.route("api/v1/get/domain/status/minimal", methods=['POST']) -@token_required('analyst') -def get_domain_status_minimal(): - data = request.get_json() - domain = data.get('domain', None) - # error handler - # TODO TO MIGRATE - raise Exception('TO MIGRATE') - # res = Domain.api_verify_if_domain_exist(domain) - if res: - return create_json_response(res[0], res[1]) - # TODO TO MIGRATE - raise Exception('TO MIGRATE') - # res = Domain.api_get_domain_up_range(domain) - res[0]['domain'] = domain - return create_json_response(res[0], res[1]) -''' - -# @restApi.route("api/v1/get/crawled/domain/list", methods=['POST']) -# @token_required('analyst') -# def get_crawled_domain_list(): -# data = request.get_json() -# res = get_mandatory_fields(data, ['date_from', 'date_to']) -# if res: -# return create_json_response(res[0], res[1]) -# -# date_from = data.get('date_from', None) -# date_to = data.get('date_to', None) -# domain_type = data.get('domain_type', None) -# domain_status = 'UP' -# # TODO TO MIGRATE -# raise Exception('TO MIGRATE') -# # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type) -# dict_res = res[0] -# dict_res['date_from'] = date_from -# dict_res['date_to'] = date_to -# dict_res['domain_status'] = domain_status -# dict_res['domain_type'] = domain_type -# return create_json_response(dict_res, res[1]) - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# -# POST JSON FORMAT -# -# { -# "type": "text", (default value) -# "tags": [], (default value) -# "default_tags": True, (default value) -# "galaxy" [], (default value) -# "text": "", mandatory if type = text -# } -# -# response: {"uuid": "uuid"} -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -''' -@restApi.route("api/v1/import/item", methods=['POST']) -@token_required('analyst') -def import_item(): - - data = request.get_json() - if not data: - return create_json_response({'status': 'error', 'reason': 'Malformed JSON'}, 400) - - # unpack json - text_to_import = data.get('text', None) - if not text_to_import: - return create_json_response({'status': 'error', 'reason': 'No text supplied'}, 400) - - tags = data.get('tags', []) - if not type(tags) is list: - tags = [] - galaxy = data.get('galaxy', []) - if not type(galaxy) is list: - galaxy = [] - - if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy): - return create_json_response({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) - - default_tags = data.get('default_tags', True) - if default_tags: - tags.append('infoleak:submission="manual"') - - if sys.getsizeof(text_to_import) > 900000: - return create_json_response({'status': 'error', 'reason': 'Size exceeds default'}, 413) - - import_uuid = str(uuid.uuid4()) - Import_helper.create_import_queue(tags, galaxy, text_to_import, import_uuid) - - return Response(json.dumps({'uuid': import_uuid}, indent=2, sort_keys=True), mimetype='application/json') - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# GET -# -# { -# "uuid": "uuid", mandatory -# } -# -# response: { -# "status": "in queue"/"in progress"/"imported", -# "items": [all item id] -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/get/import/item", methods=['POST']) -@token_required('analyst') -def import_item_uuid(): - data = request.get_json() - import_uuid = data.get('uuid', None) - - # Verify uuid - if not is_valid_uuid_v4(import_uuid): - return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 - - data = Import_helper.check_import_status(import_uuid) - if data: - return Response(json.dumps(data[0]), mimetype='application/json'), data[1] - - return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400 -''' diff --git a/var/www/modules/restApi/templates/api_default.html b/var/www/modules/restApi/templates/api_default.html deleted file mode 100644 index cb2cc24f..00000000 --- a/var/www/modules/restApi/templates/api_default.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - - AIL-Framework - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
    -
    - - {% include 'crawler/menu_sidebar.html' %} - -
    - - -
    -
    -
    - - - diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py deleted file mode 100644 index e68ee64b..00000000 --- a/var/www/modules/search/Flask_search.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - - -# import os -# import sys -# import datetime -# import flask -# from flask import Flask, render_template, jsonify, request, Blueprint -# -# from Role_Manager import login_admin, login_user_no_api -# from flask_login import login_required -# -# from whoosh import index -# from whoosh.fields import Schema, TEXT, ID -# from whoosh.qparser import QueryParser -# -# sys.path.append(os.environ['AIL_BIN']) -# ################################## -# # Import Project packages -# ################################## -# from lib.objects.Items import Item -# -# import time -# -# # ============ VARIABLES ============ -# import Flask_config -# -# app = Flask_config.app -# config_loader = Flask_config.config_loader -# baseUrl = Flask_config.baseUrl -# max_preview_char = Flask_config.max_preview_char -# max_preview_modal = Flask_config.max_preview_modal -# bootstrap_label = Flask_config.bootstrap_label -# PASTES_FOLDER = Flask_config.PASTES_FOLDER -# -# baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path")) -# indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register")) -# -# searches = Blueprint('searches', __name__, template_folder='templates') -# -# # ============ FUNCTIONS ============ -# def get_current_index(): -# with open(indexRegister_path, "r") as f: -# allIndex = f.read() -# allIndex = allIndex.split() # format [time1\ntime2] -# allIndex.sort() -# try: -# indexname = allIndex[-1].strip('\n\r') -# except IndexError as e: -# indexname = "no-index" -# indexpath = os.path.join(baseindexpath, indexname) -# return indexpath -# -# def get_index_list(selected_index=""): -# temp = [] -# index_list = [] -# for dirs in os.listdir(baseindexpath): -# if os.path.isdir(os.path.join(baseindexpath, dirs)): -# value = dirs -# name = to_iso_date(dirs) + " - " + \ -# str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \ -# #"(" + str(get_item_count(dirs))''' + " Items" + ")" -# flag = dirs==selected_index.split('/')[-1] -# if dirs == "old_index": -# temp = [value, name, flag] -# else: -# index_list.append([value, name, flag]) -# -# index_list.sort(reverse=True, key=lambda x: x[0]) -# if len(temp) != 0: -# index_list.append(temp) -# -# return index_list -# -# def get_dir_size(directory): -# cur_sum = 0 -# for directory, subdirs, files in os.walk(os.path.join(baseindexpath,directory)): -# try: -# cur_sum += sum(os.path.getsize(os.path.join(directory, name)) for name in files) -# except OSError as e: #File disappeared -# pass -# return cur_sum -# -# def get_item_count(dirs): -# ix = index.open_dir(os.path.join(baseindexpath, dirs)) -# return ix.doc_count_all() -# -# def to_iso_date(timestamp): -# if timestamp == "old_index": -# return "old_index" -# return str(datetime.datetime.fromtimestamp(int(timestamp))).split()[0] -# -# -# # ============ ROUTES ============ -# -# @searches.route("/search", methods=['POST']) -# @login_required -# @login_user_no_api -# def search(): -# query = request.form['query'] -# q = [] -# q.append(query) -# r = [] #complete path -# c = [] #preview of the paste content -# paste_date = [] -# paste_size = [] -# paste_tags = [] -# index_name = request.form['index_name'] -# num_elem_to_get = 50 -# -# # select correct index -# if index_name is None or index_name == "0": -# selected_index = get_current_index() -# else: -# selected_index = os.path.join(baseindexpath, index_name) -# -# ''' temporary disabled -# # # TODO: search by filename/item id -# ''' -# -# # Search full line -# schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) -# -# ix = index.open_dir(selected_index) -# with ix.searcher() as searcher: -# query = QueryParser("content", ix.schema).parse("".join(q)) -# results = searcher.search_page(query, 1, pagelen=num_elem_to_get) -# for x in results: -# r.append(x.items()[0][1].replace(PASTES_FOLDER, '', 1)) -# path = x.items()[0][1].replace(PASTES_FOLDER, '', 1) -# item = Item(path) -# content = item.get_content() -# content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 -# c.append(content[0:content_range]) -# curr_date = item.get_date(separator=True) -# paste_date.append(curr_date) -# paste_size.append(item.get_size()) -# p_tags = item.get_tags() -# l_tags = [] -# for tag in p_tags: -# complete_tag = tag -# tag = tag.split('=') -# if len(tag) > 1: -# if tag[1] != '': -# tag = tag[1][1:-1] -# # no value -# else: -# tag = tag[0][1:-1] -# # use for custom tags -# else: -# tag = tag[0] -# -# l_tags.append( (tag, complete_tag) ) -# -# paste_tags.append(l_tags) -# results = searcher.search(query) -# num_res = len(results) -# -# index_list = get_index_list() -# -# index_min = 1 -# index_max = len(index_list) -# -# return render_template("search.html", r=r, c=c, -# query=request.form['query'], paste_date=paste_date, -# paste_size=paste_size, char_to_display=max_preview_modal, -# num_res=num_res, index_min=index_min, index_max=index_max, -# bootstrap_label=bootstrap_label, -# paste_tags=paste_tags, -# index_list=index_list -# ) -# -# -# @searches.route("/get_more_search_result", methods=['POST']) -# @login_required -# @login_user_no_api -# def get_more_search_result(): -# query = request.form['query'] -# q = [] -# q.append(query) -# page_offset = int(request.form['page_offset']) -# index_name = request.form['index_name'] -# num_elem_to_get = 50 -# -# # select correct index -# if index_name is None or index_name == "0": -# selected_index = get_current_index() -# else: -# selected_index = os.path.join(baseindexpath, index_name) -# -# path_array = [] -# preview_array = [] -# date_array = [] -# size_array = [] -# list_tags = [] -# -# schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) -# -# ix = index.open_dir(selected_index) -# with ix.searcher() as searcher: -# query = QueryParser("content", ix.schema).parse(" ".join(q)) -# results = searcher.search_page(query, page_offset, num_elem_to_get) -# for x in results: -# path = x.items()[0][1] -# path = path.replace(PASTES_FOLDER, '', 1) -# path_array.append(path) -# item = Item(path) -# content = item.get_content() -# content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 -# preview_array.append(content[0:content_range]) -# curr_date = item.get_date(separator=True) -# date_array.append(curr_date) -# size_array.append(item.get_size()) -# p_tags = item.get_tags() -# l_tags = [] -# for tag in p_tags: -# complete_tag = tag -# tag = tag.split('=') -# if len(tag) > 1: -# if tag[1] != '': -# tag = tag[1][1:-1] -# # no value -# else: -# tag = tag[0][1:-1] -# # use for custom tags -# else: -# tag = tag[0] -# -# l_tags.append( (tag, complete_tag) ) -# list_tags.append(l_tags) -# -# to_return = {} -# to_return["path_array"] = path_array -# to_return["preview_array"] = preview_array -# to_return["date_array"] = date_array -# to_return["size_array"] = size_array -# to_return["list_tags"] = list_tags -# to_return["bootstrap_label"] = bootstrap_label -# if len(path_array) < num_elem_to_get: #pagelength -# to_return["moreData"] = False -# else: -# to_return["moreData"] = True -# -# return jsonify(to_return) -# -# -# # ========= REGISTRATION ========= -# app.register_blueprint(searches, url_prefix=baseUrl) diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html deleted file mode 100644 index 493601cc..00000000 --- a/var/www/modules/search/templates/search.html +++ /dev/null @@ -1,329 +0,0 @@ - - - - - - - - Search - AIL - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - - - - - -
    - -
    -
    -
    -
    -
    -
    - {{ r|length }} Results for "{{ query }}" -
    - -
    -
    - -
    -
    -
    - Index: - -
    -
    - - - - - - - - - - - - {% for path in r %} - - - - - - - - {% endfor %} - -
    #PathDateSize (Kb)Action
    {{ loop.index0 }}{{ path }} -
    - {% for tag in paste_tags[loop.index0] %} - - {{ tag[0] }} - - {% endfor %} -
    -     		{{ paste_date[loop.index0] }}{{ paste_size[loop.index0] }}

    -
    - - Totalling {{ num_res }} results related to paste content -
    -
    - -
    -
    - -
    - -
    - -
    - - - - - - - - - - diff --git a/var/www/modules/sentiment/Flask_sentiment.py b/var/www/modules/sentiment/Flask_sentiment.py deleted file mode 100644 index ef845fd7..00000000 --- a/var/www/modules/sentiment/Flask_sentiment.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -# ''' -# Flask functions and routes for the trending modules page -# ''' -# import os -# import sys -# import datetime -# import calendar -# import flask -# from flask import Flask, render_template, jsonify, request, Blueprint -# -# from Role_Manager import login_admin, login_read_only -# from flask_login import login_required -# -# sys.path.append(os.environ['AIL_BIN']) -# ################################## -# # Import Project packages -# ################################## -# from packages.Date import Date -# -# # ============ VARIABLES ============ -# import Flask_config -# -# app = Flask_config.app -# baseUrl = Flask_config.baseUrl -# r_serv_charts = Flask_config.r_serv_charts -# r_serv_sentiment = Flask_config.r_serv_sentiment -# -# sentiments = Blueprint('sentiments', __name__, template_folder='templates') -# -# # ============ FUNCTIONS ============ -# -# def get_date_range(num_day): -# curr_date = datetime.date.today() -# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) -# date_list = [] -# -# for i in range(0, num_day+1): -# date_list.append(date.substract_day(i)) -# return date_list -# -# -# # ============ ROUTES ============ -# -# @sentiments.route("/sentiment_analysis_trending/") -# @login_required -# @login_read_only -# def sentiment_analysis_trending(): -# return render_template("sentiment_analysis_trending.html") -# -# -# @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET']) -# @login_required -# @login_read_only -# def sentiment_analysis_getplotdata(): -# # Get the top providers based on number of pastes -# oneHour = 60*60 -# sevenDays = oneHour*24*7 -# dateStart = datetime.datetime.now() -# dateStart = dateStart.replace(minute=0, second=0, microsecond=0) -# dateStart_timestamp = calendar.timegm(dateStart.timetuple()) -# -# getAllProviders = request.args.get('getProviders') -# provider = request.args.get('provider') -# allProvider = request.args.get('all') -# if getAllProviders == 'True': -# if allProvider == "True": -# range_providers = r_serv_charts.smembers('all_provider_set') -# -# return jsonify(list(range_providers)) -# else: -# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) -# # if empty, get yesterday top providers -# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers -# -# -# # if still empty, takes from all providers -# if range_providers == []: -# print('today provider empty') -# range_providers = r_serv_charts.smembers('all_provider_set') -# -# return jsonify(list(range_providers)) -# -# elif provider is not None: -# to_return = {} -# -# cur_provider_name = provider + '_' -# list_date = {} -# for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): -# cur_set_name = cur_provider_name + str(cur_timestamp) -# -# list_value = [] -# for cur_id in r_serv_sentiment.smembers(cur_set_name): -# cur_value = (r_serv_sentiment.get(cur_id)) -# list_value.append(cur_value) -# list_date[cur_timestamp] = list_value -# to_return[provider] = list_date -# -# return jsonify(to_return) -# return "Bad request" -# -# -# -# @sentiments.route("/sentiment_analysis_plot_tool/") -# @login_required -# @login_read_only -# def sentiment_analysis_plot_tool(): -# return render_template("sentiment_analysis_plot_tool.html") -# -# -# -# @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET']) -# @login_required -# @login_read_only -# def sentiment_analysis_plot_tool_getdata(): -# getProviders = request.args.get('getProviders') -# -# if getProviders == 'True': -# providers = [] -# for cur_provider in r_serv_charts.smembers('all_provider_set'): -# providers.append(cur_provider) -# return jsonify(providers) -# -# else: -# query = request.args.get('query') -# query = query.split(',') -# Qdate = request.args.get('Qdate') -# -# date1 = (Qdate.split('-')[0]).split('/') -# date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1])) -# -# date2 = (Qdate.split('-')[1]).split('/') -# date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1])) -# -# timestamp1 = calendar.timegm(date1.timetuple()) -# timestamp2 = calendar.timegm(date2.timetuple()) -# -# oneHour = 60*60 -# oneDay = oneHour*24 -# -# to_return = {} -# for cur_provider in query: -# list_date = {} -# cur_provider_name = cur_provider + '_' -# for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour): -# cur_set_name = cur_provider_name + str(cur_timestamp) -# -# list_value = [] -# for cur_id in r_serv_sentiment.smembers(cur_set_name): -# cur_value = (r_serv_sentiment.get(cur_id)) -# list_value.append(cur_value) -# list_date[cur_timestamp] = list_value -# to_return[cur_provider] = list_date -# -# return jsonify(to_return) -# -# # ========= REGISTRATION ========= -# app.register_blueprint(sentiments, url_prefix=baseUrl) diff --git a/var/www/modules/sentiment/templates/header_sentiment.html b/var/www/modules/sentiment/templates/header_sentiment.html deleted file mode 100644 index 7696c9a7..00000000 --- a/var/www/modules/sentiment/templates/header_sentiment.html +++ /dev/null @@ -1,7 +0,0 @@ -{#
  • Sentiment Analysis#} -{# #} -{# #} -{#
    • #} diff --git a/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html b/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html deleted file mode 100644 index 78f32976..00000000 --- a/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - - Sentiment Plot Tool - AIL - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    -
    -

    Sentiment analysis: Plot tool

    -
    - -
    - -
    - - -
    -
    -
    -
    -
    -
    - Select options -
    -
    -
    - -
    - -
    -
    -
      -
    -
    -
    -
    -
    -
      -
    -
    -
    -
    - -
    -
    - Date: -
    -
    - -
    - - -
    -
    - -
    -
    - -
    - -
    -
    - - -
    -
    -
    -
    -
    -
    - Graph -
    -
    -
    -
    - -
    -
    - -
    - -
    -
    - - - -
    - -
    - - - - - - - - diff --git a/var/www/modules/sentiment/templates/sentiment_analysis_trending.html b/var/www/modules/sentiment/templates/sentiment_analysis_trending.html deleted file mode 100644 index 13a6c14b..00000000 --- a/var/www/modules/sentiment/templates/sentiment_analysis_trending.html +++ /dev/null @@ -1,251 +0,0 @@ - - - - - - - - Sentiment Trending - AIL - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    -
    -

    Sentiment analysis: Trending

    -
    - -
    - -
    - - - -
    -
    -
    -
    -
    -
    - Today's mood -
    -
    - -
    - -
    -
    - -
    -
    -
    -
    - -
    -
    -
    - -
    -
    - Mood value -
    - Compound by mood -
    -
    -
    - Mood value -
    - Compound by mood -
    -
    -
    - -
    - -
    -
    - -
    - -
    -
    - - -
    -
    -
    -
    -
    -
    - Week's mood -
    -
    - -
    - -
    -
    - -
    -
    -
    -
    - -
    -
    -
    - -
    -
    - Mood value -
    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Worst moodBest mood
    no datano data
    no datano data
    no datano data
    no datano data
    no datano data
    -
    -
    - -
    - -
    -
    - -
    - -
    -
    - - - -
    - -
    - - - - - - - - diff --git a/var/www/modules/trendingcharts/Flask_trendingcharts.py b/var/www/modules/trendingcharts/Flask_trendingcharts.py deleted file mode 100644 index f00a9bad..00000000 --- a/var/www/modules/trendingcharts/Flask_trendingcharts.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending charts page -''' -# import os -# import sys -# import datetime -# import flask -# from flask import Flask, render_template, jsonify, request, Blueprint -# -# from Role_Manager import login_admin, login_read_only -# from flask_login import login_required -# -# sys.path.append(os.environ['AIL_BIN']) -# ################################## -# # Import Project packages -# ################################## -# from packages.Date import Date -# -# # ============ VARIABLES ============ -# import Flask_config -# -# app = Flask_config.app -# config_loader = Flask_config.config_loader -# baseUrl = Flask_config.baseUrl -# r_serv_charts = Flask_config.r_serv_charts -# -# trendings = Blueprint('trendings', __name__, template_folder='templates') -# -# # ============ FUNCTIONS ============ -# -# def get_date_range(num_day): -# curr_date = datetime.date.today() -# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) -# date_list = [] -# -# for i in range(0, num_day+1): -# date_list.append(date.substract_day(i)) -# -# return date_list -# -# -# # ============ ROUTES ============ -# -# @trendings.route("/_progressionCharts", methods=['GET']) -# @login_required -# @login_read_only -# def progressionCharts(): -# attribute_name = request.args.get('attributeName') -# trending_name = request.args.get('trendingName') -# bar_requested = True if request.args.get('bar') == "true" else False -# -# if (bar_requested): -# num_day = int(request.args.get('days')) -# bar_values = [] -# -# date_range = get_date_range(num_day) -# # Retreive all data from the last num_day -# for date in date_range: -# -# curr_value = r_serv_charts.hget(attribute_name, date) -# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) -# bar_values.insert(0, attribute_name) -# return jsonify(bar_values) -# -# else: -# redis_progression_name = "z_top_progression_" + trending_name -# keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10) -# -# return jsonify(keyw_value) -# -# @trendings.route("/wordstrending/") -# @login_required -# @login_read_only -# def wordstrending(): -# default_display = config_loader.get_config_str("Flask", "default_display") -# return render_template("Wordstrending.html", default_display = default_display) -# -# -# @trendings.route("/protocolstrending/") -# @login_required -# @login_read_only -# def protocolstrending(): -# default_display = config_loader.get_config_str("Flask", "default_display") -# return render_template("Protocolstrending.html", default_display = default_display) -# -# -# @trendings.route("/trending/") -# @login_required -# @login_read_only -# def trending(): -# default_display = config_loader.get_config_str("Flask", "default_display") -# return render_template("Trending.html", default_display = default_display) -# -# -# # ========= REGISTRATION ========= -# app.register_blueprint(trendings, url_prefix=baseUrl) diff --git a/var/www/modules/trendingcharts/templates/Trending.html b/var/www/modules/trendingcharts/templates/Trending.html deleted file mode 100644 index 9e3728ca..00000000 --- a/var/www/modules/trendingcharts/templates/Trending.html +++ /dev/null @@ -1,154 +0,0 @@ - - - - - - - - - - - - Trending Charts - AIL - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    -
    -

    Trending charts

    -
    - -
    - -
    - - - -
    - - - - - -
    -
    - {% include 'trending_graphs/Tldstrending.html' %} -
    -
    - {% include 'trending_graphs/Domainstrending.html' %} -
    -
    - {% include 'trending_graphs/Protocolstrending.html' %} -
    -
    - {% include 'trending_graphs/Wordstrending.html' %} -
    -
    - -
    - - - - - - - - - - - - -
    - - - - diff --git a/var/www/modules/trendingcharts/templates/trending_graphs/Domainstrending.html b/var/www/modules/trendingcharts/templates/trending_graphs/Domainstrending.html deleted file mode 100644 index 09052204..00000000 --- a/var/www/modules/trendingcharts/templates/trending_graphs/Domainstrending.html +++ /dev/null @@ -1,82 +0,0 @@ -
    -
    -
    -
    -
    -
    - Top Progression for the last 5 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    - -
    -
    -
    - Top Progression for the last 15 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    -
    - -
    -
    - - -
    -
    - Top Domain Trending -
    -
    - - -
    -
    - -
    - - - -
    - -
    - -
    - -
    -
    - -
    diff --git a/var/www/modules/trendingcharts/templates/trending_graphs/Moduletrending.html b/var/www/modules/trendingcharts/templates/trending_graphs/Moduletrending.html deleted file mode 100644 index 6ef2303e..00000000 --- a/var/www/modules/trendingcharts/templates/trending_graphs/Moduletrending.html +++ /dev/null @@ -1,96 +0,0 @@ -
    -
    -
    -
    -
    -
    - Credential - most posted domain - Today -
    -
    -
    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    - -
    - -
    -
    - -
    -
    -
    -
    -
    -
    - Mail - most posted domain (max 1 per paste) - Today -
    -
    -
    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    - -
    - -
    -
    - -
    -
    -
    -
    -
    -
    - Provider - Today -
    -
    -
    -
    -

    Average paste size by provider

    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    -

    Number of paste by provider

    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    - -
    - -
    -
    - -
    - -
    -
    diff --git a/var/www/modules/trendingcharts/templates/trending_graphs/Protocolstrending.html b/var/www/modules/trendingcharts/templates/trending_graphs/Protocolstrending.html deleted file mode 100644 index c0f484c7..00000000 --- a/var/www/modules/trendingcharts/templates/trending_graphs/Protocolstrending.html +++ /dev/null @@ -1,82 +0,0 @@ -
    -
    -
    -
    -
    -
    - Top Progression for the last 5 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    - -
    -
    -
    - Top Progression for the last 15 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    -
    - -
    -
    - - -
    -
    - Protocols Trend -
    -
    - - -
    -
    - -
    - - - -
    - -
    - -
    - -
    -
    - -
    diff --git a/var/www/modules/trendingcharts/templates/trending_graphs/Tldstrending.html b/var/www/modules/trendingcharts/templates/trending_graphs/Tldstrending.html deleted file mode 100644 index 6d1590e4..00000000 --- a/var/www/modules/trendingcharts/templates/trending_graphs/Tldstrending.html +++ /dev/null @@ -1,84 +0,0 @@ -
    -
    -
    -
    -
    -
    - Top Progression for the last 5 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    -
    Click on a part
    -
    -
    -
    - -
    - -
    -
    - -
    -
    -
    - Top Progression for the last 15 days -
    -
    -
    -
    No bar hovered
    -
    -
    -
    Click on a part
    -
    - -
    - -
    -
    -
    - -
    -
    - - -
    -
    - Top Level Domain Trending - -
    -
    - - -
    -
    -
    - - - -
    - -
    - -
    - -
    -
    - -
    diff --git a/var/www/modules/trendingcharts/templates/trending_graphs/Wordstrending.html b/var/www/modules/trendingcharts/templates/trending_graphs/Wordstrending.html deleted file mode 100644 index 38dd8d87..00000000 --- a/var/www/modules/trendingcharts/templates/trending_graphs/Wordstrending.html +++ /dev/null @@ -1,37 +0,0 @@ -
    -
    - Words Trend -
    -
    - - -
    -
    - -
    - - - -
    - -
    - -
    - -
    -
    - -
    diff --git a/var/www/modules/trendingmodules/Flask_trendingmodules.py b/var/www/modules/trendingmodules/Flask_trendingmodules.py deleted file mode 100644 index 18e07ad4..00000000 --- a/var/www/modules/trendingmodules/Flask_trendingmodules.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending modules page -''' -# import os -# import sys -# import datetime -# import flask -# from flask import Flask, render_template, jsonify, request, Blueprint -# -# from Role_Manager import login_admin, login_read_only -# from flask_login import login_required -# -# sys.path.append(os.environ['AIL_BIN']) -# ################################## -# # Import Project packages -# ################################## -# from packages.Date import Date -# -# # ============ VARIABLES ============ -# import Flask_config -# -# app = Flask_config.app -# baseUrl = Flask_config.baseUrl -# r_serv_charts = Flask_config.r_serv_charts -# -# trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates') -# -# # ============ FUNCTIONS ============ -# -# # Iterate over elements in the module provided and return the today data or the last data -# # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]] -# def get_top_relevant_data(server, module_name): -# days = 0 -# for date in get_date_range(15): -# redis_progression_name_set = 'top_'+ module_name +'_set_' + date -# member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True) -# -# if len(member_set) == 0: #No data for this date -# days += 1 -# else: -# member_set.insert(0, ("passed_days", days)) -# return member_set -# -# -# def get_date_range(num_day): -# curr_date = datetime.date.today() -# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) -# date_list = [] -# -# for i in range(0, num_day+1): -# date_list.append(date.substract_day(i)) -# return date_list -# -# # ============ ROUTES ============ -# -# @trendingmodules.route("/_moduleCharts", methods=['GET']) -# @login_required -# @login_read_only -# def modulesCharts(): -# keyword_name = request.args.get('keywordName') -# module_name = request.args.get('moduleName') -# bar_requested = True if request.args.get('bar') == "true" else False -# -# if (bar_requested): -# num_day = int(request.args.get('days')) -# bar_values = [] -# -# date_range = get_date_range(num_day) -# # Retreive all data from the last num_day -# for date in date_range: -# curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name) -# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) -# bar_values.insert(0, keyword_name) -# return jsonify(bar_values) -# -# else: -# member_set = get_top_relevant_data(r_serv_charts, module_name) -# member_set = member_set if member_set is not None else [] -# if len(member_set) == 0: -# member_set.append(("No relevant data", int(100))) -# return jsonify(member_set) -# -# -# @trendingmodules.route("/_providersChart", methods=['GET']) -# @login_required -# @login_read_only -# def providersChart(): -# keyword_name = request.args.get('keywordName') -# module_name = request.args.get('moduleName') -# bar_requested = True if request.args.get('bar') == "true" else False -# -# if (bar_requested): -# num_day = int(request.args.get('days')) -# bar_values = [] -# -# date_range = get_date_range(num_day) -# # Retreive all data from the last num_day -# for date in date_range: -# curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) ) -# if curr_value_size is not None: -# curr_value_size = curr_value_size -# -# curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date) -# -# curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date) -# if curr_value_size_avg is not None: -# curr_value_size_avg = curr_value_size_avg -# -# -# if module_name == "size": -# curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0) -# else: -# curr_value = float(curr_value_num if curr_value_num is not None else 0.0) -# -# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value]) -# bar_values.insert(0, keyword_name) -# return jsonify(bar_values) -# -# else: -# #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set' -# redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_' -# redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0] -# -# member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8) -# -# # Member set is a list of (value, score) pairs -# if len(member_set) == 0: -# member_set.append(("No relevant data", float(100))) -# return jsonify(member_set) -# -# -# @trendingmodules.route("/moduletrending/") -# @login_required -# @login_read_only -# def moduletrending(): -# return render_template("Moduletrending.html") -# -# -# # ========= REGISTRATION ========= -# app.register_blueprint(trendingmodules, url_prefix=baseUrl) diff --git a/var/www/modules/trendingmodules/templates/Moduletrending.html b/var/www/modules/trendingmodules/templates/Moduletrending.html deleted file mode 100644 index 7e4a3085..00000000 --- a/var/www/modules/trendingmodules/templates/Moduletrending.html +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - - - Modules Statistics - AIL - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    - {% include 'trending_graphs/Moduletrending.html' %} -
    - - - - - - - - - - - - diff --git a/var/www/modules/trendingmodules/templates/header_trendingmodules.html b/var/www/modules/trendingmodules/templates/header_trendingmodules.html deleted file mode 100644 index 2d3d2727..00000000 --- a/var/www/modules/trendingmodules/templates/header_trendingmodules.html +++ /dev/null @@ -1,3 +0,0 @@ -{# -
  • Modules statistics
    • -#} diff --git a/var/www/templates/base_template.html b/var/www/templates/base_template.html deleted file mode 100644 index 46d6fb1e..00000000 --- a/var/www/templates/base_template.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - Analysis Information Leak framework Dashboard - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    -
    -

    MODULENAME

    -
    - -
    - - -
    - - - - - - diff --git a/var/www/templates/navbar.html b/var/www/templates/navbar.html deleted file mode 100644 index 27543106..00000000 --- a/var/www/templates/navbar.html +++ /dev/null @@ -1,19 +0,0 @@ -
    -
    - {% include 'header.html' %} - - - -
    -
    diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 01d86cdf..39f49b7a 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -8,7 +8,7 @@ git submodule update wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js BOOTSTRAP_VERSION='4.2.1' -FONT_AWESOME_VERSION='5.7.1' +FONT_AWESOME_VERSION='6.6.0' D3_JS_VERSION='5.16.0' wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js @@ -17,8 +17,7 @@ rm -rf temp mkdir temp wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip -wget https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip -wget https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip # dateRangePicker @@ -26,8 +25,7 @@ wget https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip wget https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ -unzip -qq temp/FONT_AWESOME_4.7.0.zip -d temp/ -unzip -qq temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ +unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/ unzip -qq temp/moment.zip -d temp/ @@ -38,8 +36,6 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/bootstrap4.min.css mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/bootstrap4.min.css.map -mv temp/Font-Awesome-4.7.0 temp/font-awesome - rm -rf ./static/webfonts/ mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts