From ae6f8af09fba3fc83d7e1d761ab79ed07f7c128a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 28 Feb 2023 11:01:27 +0100 Subject: [PATCH] chg: [subtype objects] migrate UI cryptocurrency, pgp, username --- bin/lib/ail_core.py | 3 + bin/lib/objects/CryptoCurrencies.py | 2 +- bin/lib/objects/Pgps.py | 2 +- bin/lib/objects/Usernames.py | 2 +- bin/lib/objects/abstract_subtype_object.py | 82 ++- bin/lib/objects/ail_objects.py | 2 +- bin/packages/Date.py | 10 + var/www/Flask_server.py | 2 + var/www/blueprints/correlation.py | 10 +- var/www/blueprints/objects_subtypes.py | 166 +++++ var/www/modules/Flask_config.py | 5 +- .../modules/hashDecoded/Flask_hashDecoded.py | 544 -------------- .../hashDecoded/templates/hashDecoded.html | 676 ------------------ .../templates/header_hashDecoded.html | 1 - .../correlation/show_correlation.html | 6 +- var/www/templates/decoded/menu_sidebar.html | 60 -- .../objects/subtypes_objs_dashboard.html} | 117 +-- .../templates/sidebars/sidebar_objects.html | 6 +- 18 files changed, 332 insertions(+), 1364 deletions(-) create mode 100644 var/www/blueprints/objects_subtypes.py delete mode 100644 var/www/modules/hashDecoded/Flask_hashDecoded.py delete mode 100644 var/www/modules/hashDecoded/templates/hashDecoded.html delete mode 100644 var/www/modules/hashDecoded/templates/header_hashDecoded.html delete mode 100644 var/www/templates/decoded/menu_sidebar.html rename var/www/{modules/hashDecoded/templates/DaysCorrelation.html => templates/objects/subtypes_objs_dashboard.html} (78%) diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 923d16f0..26f52179 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -25,6 +25,9 @@ def get_ail_uuid(): def get_all_objects(): return AIL_OBJECTS +def get_objects_with_subtypes(): + return ['cryptocurrency', 'pgp', 'username'] + def get_object_all_subtypes(obj_type): if obj_type == 'cryptocurrency': return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index cabe3d8a..96073ed1 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -120,7 +120,7 @@ class CryptoCurrency(AbstractSubtypeObject): return obj def get_meta(self, options=set()): - meta = self._get_meta() + meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index cf169850..201b7226 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -43,7 +43,7 @@ class Pgp(AbstractSubtypeObject): # # TODO: def get_meta(self, options=set()): - meta = self._get_meta() + meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index 1e0783f0..b4226d28 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -64,7 +64,7 @@ class Username(AbstractSubtypeObject): return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} def get_meta(self, options=set()): - meta = self._get_meta() + meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 6a8122c1..bc466473 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -8,6 +8,7 @@ Base Class for AIL Objects ################################## import os import sys +from abc import ABC # from flask import url_for @@ -16,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects.abstract_object import AbstractObject +from lib.ail_core import get_object_all_subtypes from lib.ConfigLoader import ConfigLoader from lib.item_basic import is_crawled, get_item_domain from lib.data_retention_engine import update_obj_date @@ -31,7 +33,7 @@ config_loader = None # # FIXME: SAVE SUBTYPE NAMES ????? -class AbstractSubtypeObject(AbstractObject): +class AbstractSubtypeObject(AbstractObject, ABC): """ Abstract Subtype Object """ @@ -80,11 +82,19 @@ class AbstractSubtypeObject(AbstractObject): else: return int(nb) - def _get_meta(self): - meta_dict = {'first_seen': self.get_first_seen(), - 'last_seen': self.get_last_seen(), - 'nb_seen': self.get_nb_seen()} - return meta_dict + def _get_meta(self, options=None): + if options is None: + options = set() + meta = {'first_seen': self.get_first_seen(), + 'last_seen': self.get_last_seen(), + 'nb_seen': self.get_nb_seen()} + if 'icon' in options: + meta['icon'] = self.get_svg_icon() + if 'link' in options: + meta['link'] = self.get_link() + if 'sparkline' in options: + meta['sparkline'] = self.get_sparkline() + return meta def set_first_seen(self, first_seen): r_object.hset(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen', first_seen) @@ -111,6 +121,17 @@ class AbstractSubtypeObject(AbstractObject): for date in Date.get_previous_date_list(6): sparkline.append(self.get_nb_seen_by_date(date)) return sparkline + + def get_graphline(self, date_from=None, date_to=None): + graphline = [] + # TODO get by daterange + # if date_from and date_to: + dates = Date.get_date_range(30) + for date in dates: + nb = self.get_nb_seen_by_date(date) + date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}' + graphline.append({'date': date, 'value': nb}) + return graphline # # HANDLE Others objects ???? # @@ -151,3 +172,52 @@ class AbstractSubtypeObject(AbstractObject): def get_all_id(obj_type, subtype): return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1) + +def get_subtypes_objs_by_date(obj_type, subtype, date): + return r_object.hkeys(f'{obj_type}:{subtype}:{date}') + +def get_subtypes_objs_by_daterange(obj_type, date_from, date_to, subtype=None): + if subtype: + subtypes = [subtype] + else: + subtypes = get_object_all_subtypes(obj_type) + objs = set() + for date in Date.get_daterange(date_from, date_to): + for subtype in subtypes: + for obj_id in get_subtypes_objs_by_date(obj_type, subtype, date): + objs.add((obj_type, subtype, obj_id)) + return objs + + +def get_subtypes_objs_range_json(obj_type, date_from, date_to): + objs_range = [] + dates = Date.get_daterange(date_from, date_to) + if len(dates) == 1: + dict_subtype = {} + subtypes = get_object_all_subtypes(obj_type) + for subtype in subtypes: + dict_subtype[subtype] = 0 + for subtype in get_object_all_subtypes(obj_type): + day_dict = dict_subtype.copy() + day_dict['date'] = subtype + # if don't filter duplicates + # nb = 0 + # for val in r_object.hvals(f'{obj_type}:{subtype}:{dates[0]}'): + # nb += int(val) + # day_dict[subtype] = nb + day_dict[subtype] = r_object.hlen(f'{obj_type}:{subtype}:{dates[0]}') + objs_range.append(day_dict) + else: + subtypes = get_object_all_subtypes(obj_type) + for date in dates: + day_dict = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'} + for subtype in subtypes: + # if don't filter duplicates + # nb = 0 + # for val in r_object.hvals(f'{obj_type}:{subtype}:{date}'): + # nb += int(val) + # day_dict[subtype] = nb + day_dict[subtype] = r_object.hlen(f'{obj_type}:{subtype}:{date}') + objs_range.append(day_dict) + + return objs_range diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 88737034..4525d033 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -130,7 +130,7 @@ def add_obj_tags(obj_type, subtype, id, tags): # -TAGS- # -def get_object_meta(obj_type, subtype, id, options=[], flask_context=False): +def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta(options=options) meta['icon'] = obj.get_svg_icon() diff --git a/bin/packages/Date.py b/bin/packages/Date.py index bc68caf3..5c4b3834 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -129,6 +129,16 @@ def substract_date(date_from, date_to): l_date.append( date.strftime('%Y%m%d') ) return l_date +def get_daterange(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append(date.strftime('%Y%m%d')) + return l_date + def validate_str_date(str_date, separator=''): try: datetime.datetime.strptime(str_date, '%Y{}%m{}%d'.format(separator, separator)) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index ff4d727f..a2d491a6 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -48,6 +48,7 @@ from blueprints.ail_2_ail_sync import ail_2_ail_sync from blueprints.settings_b import settings_b from blueprints.objects_cve import objects_cve from blueprints.objects_decoded import objects_decoded +from blueprints.objects_subtypes import objects_subtypes Flask_dir = os.environ['AIL_FLASK'] @@ -109,6 +110,7 @@ app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl) app.register_blueprint(settings_b, url_prefix=baseUrl) app.register_blueprint(objects_cve, url_prefix=baseUrl) app.register_blueprint(objects_decoded, url_prefix=baseUrl) +app.register_blueprint(objects_subtypes, url_prefix=baseUrl) # ========= =========# # ========= Cookie name ======== diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 3d0db8ac..ba9c1ba7 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -163,7 +163,7 @@ def get_description(): # # TODO: return error json if not ail_objects.exists_obj(object_type, type_id, correlation_id): return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 - # oject exist + # object exist else: res = ail_objects.get_object_meta(object_type, type_id, correlation_id, flask_context=True) return jsonify(res) @@ -187,11 +187,3 @@ def graph_node_json(): #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) -@correlation.route('/correlation/subtype_search', methods=['POST']) -@login_required -@login_read_only -def subtype_search(): - obj_type = request.form.get('object_type') - obj_subtype = request.form.get('object_subtype') - obj_id = request.form.get('object_id') - return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=obj_subtype, id=obj_id)) diff --git a/var/www/blueprints/objects_subtypes.py b/var/www/blueprints/objects_subtypes.py new file mode 100644 index 00000000..c5b0fcd0 --- /dev/null +++ b/var/www/blueprints/objects_subtypes.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.objects import abstract_subtype_object +from lib.objects import ail_objects +from lib.objects import CryptoCurrencies +from packages import Date + +# ============ BLUEPRINT ============ +objects_subtypes = Blueprint('objects_subtypes', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +# TODO VERIFY SUBTYPE +def subtypes_objects_dashboard(obj_type, f_request): + if request.method == 'POST': + date_from = f_request.form.get('from') + date_to = f_request.form.get('to') + subtype = f_request.form.get('subtype') + show_objects = bool(f_request.form.get('show_objects')) + endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{obj_type}') + endpoint_dashboard = f'{endpoint_dashboard}?from={date_from}&to={date_to}' + if subtype: + if subtype == 'All types': + subtype = None + if subtype: + if not ail_objects.is_valid_object_subtype(obj_type, subtype): + subtype = None + if subtype: + endpoint_dashboard = f'{endpoint_dashboard}&subtype={subtype}' + if show_objects: + endpoint_dashboard = f'{endpoint_dashboard}&show_objects={show_objects}' + return redirect(endpoint_dashboard) + else: + date_from = f_request.args.get('from') + date_to = f_request.args.get('to') + subtype = f_request.args.get('subtype') + show_objects = bool(f_request.args.get('show_objects')) + # Date + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + daily_type_chart = date_from == date_to + # Subtype + if subtype == 'All types': + subtype = None + if subtype: + if not ail_objects.is_valid_object_subtype(obj_type, subtype): + subtype = None + + objs = [] + if show_objects: + subtypes_objs = abstract_subtype_object.get_subtypes_objs_by_daterange(obj_type, date_from, date_to, + subtype=subtype) + if subtypes_objs: + for obj_t, obj_subtype, obj_id in subtypes_objs: + objs.append(ail_objects.get_object_meta(obj_t, obj_subtype, obj_id, options={'sparkline'})) + + endpoint_dashboard = f'objects_subtypes.objects_dashboard_{obj_type}' + return render_template('subtypes_objs_dashboard.html', date_from=date_from, date_to=date_to, + daily_type_chart = daily_type_chart, show_objects=show_objects, + obj_type=obj_type, subtype=subtype, objs=objs, + subtypes = ail_core.get_object_all_subtypes(obj_type), + endpoint_dashboard=endpoint_dashboard) + + +# ============= ROUTES ============== + +@objects_subtypes.route("/objects/cryptocurrencies", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_cryptocurrency(): + return subtypes_objects_dashboard('cryptocurrency', request) + +@objects_subtypes.route("/objects/pgps", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_pgp(): + return subtypes_objects_dashboard('pgp', request) + +@objects_subtypes.route("/objects/usernames", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_username(): + return subtypes_objects_dashboard('username', request) + +# TODO REDIRECT +@objects_subtypes.route("/objects/subtypes/post", methods=['POST']) +@login_required +@login_read_only +def objects_subtypes_dashboard_post(): + obj_type = request.form.get('obj_type') + if obj_type not in ail_core.get_objects_with_subtypes(): + return create_json_response({'error': 'Invalid Object type'}, 400) + return subtypes_objects_dashboard(obj_type, request) + +@objects_subtypes.route("/objects/subtypes/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_subtypes_range_json(): + obj_type = request.args.get('type') + if obj_type not in ail_core.get_objects_with_subtypes(): + return create_json_response({'error': 'Invalid Object type'}, 400) + date_from = request.args.get('from') + date_to = request.args.get('to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + return jsonify(abstract_subtype_object.get_subtypes_objs_range_json(obj_type, date_from, date_to)) + +@objects_subtypes.route("/objects/subtypes/search", methods=['POST']) +@login_required +@login_read_only +def objects_subtypes_search(): + obj_type = request.form.get('type') + subtype = request.form.get('subtype') + obj_id = request.form.get('id') + if obj_type not in ail_core.get_objects_with_subtypes(): + return create_json_response({'error': 'Invalid Object type'}, 400) + obj = ail_objects.get_object(obj_type, subtype, obj_id) + if not obj.exists(): + abort(404) + else: + # TODO Search object + return redirect(obj.get_link(flask_context=True)) + +@objects_subtypes.route("/objects/subtypes/graphline/json", methods=['GET']) +@login_required +@login_read_only +def objects_cve_graphline_json(): + obj_type = request.args.get('type') + subtype = request.args.get('subtype') + obj_id = request.args.get('id') + if obj_type not in ail_core.get_objects_with_subtypes(): + return create_json_response({'error': 'Invalid Object type'}, 400) + obj = ail_objects.get_object(obj_type, subtype, obj_id) + if not obj.exists(): + abort(404) + else: + return jsonify(obj.get_graphline()) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index ed685fc6..2524ddcb 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -31,12 +31,11 @@ r_serv_log = config_loader.get_redis_conn("Redis_Log") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs -r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # -> TODO MIGRATE /correlation/ subtypes objects r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER # # # # # # # -r_serv_db = config_loader.get_db_conn("Kvrocks_DB") -r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") +r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint +r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint # Logger (Redis) redis_logger = publisher diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py deleted file mode 100644 index 42ed7e3e..00000000 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ /dev/null @@ -1,544 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending modules page -''' -import os -import sys -import datetime - -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file -from Role_Manager import login_admin, login_analyst, login_read_only -from flask_login import login_required - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib.objects import ail_objects - -from packages.Date import Date - -# ============ VARIABLES ============ -import Flask_config - -app = Flask_config.app -baseUrl = Flask_config.baseUrl -r_serv_metadata = Flask_config.r_serv_metadata -vt_enabled = Flask_config.vt_enabled -vt_auth = Flask_config.vt_auth -PASTES_FOLDER = Flask_config.PASTES_FOLDER - -hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates') - -## TODO: put me in option -all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'] -all_pgpdump = ['key', 'name', 'mail'] -all_username = ['telegram', 'twitter', 'jabber'] - -# ============ FUNCTIONS ============ - -def get_date_range(num_day): - curr_date = datetime.date.today() - date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) - date_list = [] - - for i in range(0, num_day+1): - date_list.append(date.substract_day(i)) - - return list(reversed(date_list)) - -def substract_date(date_from, date_to): - date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) - date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) - delta = date_to - date_from # timedelta - l_date = [] - for i in range(delta.days + 1): - date = date_from + datetime.timedelta(i) - l_date.append( date.strftime('%Y%m%d') ) - return l_date - - -def get_icon(correlation_type, type_id): - icon_text = 'fas fa-sticky-note' - if correlation_type == 'pgpdump': - # set type_id icon - if type_id == 'key': - icon_text = 'fas fa-key' - elif type_id == 'name': - icon_text = 'fas fa-user-tag' - elif type_id == 'mail': - icon_text = 'fas fa-at' - else: - icon_text = 'times' - elif correlation_type == 'cryptocurrency': - if type_id == 'bitcoin': - icon_text = 'fab fa-btc' - elif type_id == 'monero': - icon_text = 'fab fa-monero' - elif type_id == 'ethereum': - icon_text = 'fab fa-ethereum' - else: - icon_text = 'fas fa-coins' - elif correlation_type == 'username': - if type_id == 'telegram': - icon_text = 'fab fa-telegram-plane' - elif type_id == 'twitter': - icon_text = 'fab fa-twitter' - elif type_id == 'jabber': - icon_text = 'fas fa-user' - return icon_text - -def get_icon_text(correlation_type, type_id): - icon_text = '\uf249' - if correlation_type == 'pgpdump': - if type_id == 'key': - icon_text = '\uf084' - elif type_id == 'name': - icon_text = '\uf507' - elif type_id == 'mail': - icon_text = '\uf1fa' - else: - icon_text = 'times' - elif correlation_type == 'cryptocurrency': - if type_id == 'bitcoin': - icon_text = '\uf15a' - elif type_id == 'monero': - icon_text = '\uf3d0' - elif type_id == 'ethereum': - icon_text = '\uf42e' - else: - icon_text = '\uf51e' - elif correlation_type == 'username': - if type_id == 'telegram': - icon_text = '\uf2c6' - elif type_id == 'twitter': - icon_text = '\uf099' - elif type_id == 'jabber': - icon_text = '\uf007' - return icon_text - -def get_all_types_id(correlation_type): - if correlation_type == 'pgpdump': - return all_pgpdump - elif correlation_type == 'cryptocurrency': - return all_cryptocurrency - elif correlation_type == 'username': - return all_username - else: - return [] - -def get_key_id_metadata(obj_type, subtype, obj_id): - obj = ail_objects.get_object_meta(obj_type, subtype, obj_id) - return obj - -def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id): - sparklines_value = [] - for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date_day), key_id) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - sparklines_value.append(int(nb_seen_this_day)) - return sparklines_value - -def get_correlation_type_search_endpoint(correlation_type): - if correlation_type == 'pgpdump': - endpoint = 'hashDecoded.all_pgpdump_search' - elif correlation_type == 'cryptocurrency': - endpoint = 'hashDecoded.all_cryptocurrency_search' - elif correlation_type == 'username': - endpoint = 'hashDecoded.all_username_search' - else: - endpoint = 'hashDecoded.hashDecoded_page' - return endpoint - -def get_correlation_type_page_endpoint(correlation_type): - if correlation_type == 'pgpdump': - endpoint = 'hashDecoded.pgpdump_page' - elif correlation_type == 'cryptocurrency': - endpoint = 'hashDecoded.cryptocurrency_page' - elif correlation_type == 'username': - endpoint = 'hashDecoded.username_page' - else: - endpoint = 'hashDecoded.hashDecoded_page' - return endpoint - -def get_show_key_id_endpoint(correlation_type): - return 'correlation.show_correlation' - -def get_range_type_json_endpoint(correlation_type): - if correlation_type == 'pgpdump': - endpoint = 'hashDecoded.pgpdump_range_type_json' - elif correlation_type == 'cryptocurrency': - endpoint = 'hashDecoded.cryptocurrency_range_type_json' - elif correlation_type == 'username': - endpoint = 'hashDecoded.username_range_type_json' - else: - endpoint = 'hashDecoded.hashDecoded_page' - return endpoint - -############ CORE CORRELATION ############ - -def main_correlation_page(correlation_type, type_id, date_from, date_to, show_decoded_files): - - if type_id == 'All types': - type_id = None - - # verify type input - if type_id is not None: - #retrieve char - type_id = type_id.replace(' ', '') - if not ail_objects.is_valid_object_subtype(correlation_type, type_id): - type_id = None - - date_range = [] - if date_from is not None and date_to is not None: - #change format - try: - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - except: - pass - - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] - date_to = date_from - - else: - date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] - date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] - - # display day type bar chart - if len(date_range) == 1 and type is None: - daily_type_chart = True - daily_date = date_range[0] - else: - daily_type_chart = False - daily_date = None - - if type_id is None: - all_type_id = get_all_types_id(correlation_type) - else: - all_type_id = type_id - - l_keys_id_dump = set() - if show_decoded_files: - for date in date_range: - if isinstance(all_type_id, str): - l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, all_type_id, date)) - if l_dump: - for dump in l_dump: - l_keys_id_dump.add( (dump, all_type_id) ) - else: - for typ_id in all_type_id: - l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, typ_id, date)) - if l_dump: - for dump in l_dump: - l_keys_id_dump.add( (dump, typ_id) ) - - - num_day_sparkline = 6 - date_range_sparkline = get_date_range(num_day_sparkline) - - sparkline_id = 0 - keys_id_metadata = {} - for dump_res in l_keys_id_dump: - new_key_id, typ_id = dump_res - - keys_id_metadata[new_key_id] = get_key_id_metadata(correlation_type, typ_id, new_key_id) - - if keys_id_metadata[new_key_id]: - keys_id_metadata[new_key_id]['type_id'] = typ_id - keys_id_metadata[new_key_id]['type_icon'] = get_icon(correlation_type, typ_id) - - keys_id_metadata[new_key_id]['sparklines_data'] = list_sparkline_type_id_values(date_range_sparkline, correlation_type, typ_id, new_key_id) - keys_id_metadata[new_key_id]['sparklines_id'] = sparkline_id - sparkline_id += 1 - - l_type = get_all_types_id(correlation_type) - - correlation_type_n = correlation_type - if correlation_type_n=='pgpdump': - correlation_type_n = 'pgp' - - return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata, - correlation_type=correlation_type, - correlation_type_n=correlation_type_n, - correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type), - correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type), - show_key_id_endpoint=get_show_key_id_endpoint(correlation_type), - range_type_json_endpoint=get_range_type_json_endpoint(correlation_type), - l_type=l_type, type_id=type_id, - daily_type_chart=daily_type_chart, daily_date=daily_date, - date_from=date_from, date_to=date_to, - show_decoded_files=show_decoded_files) - - - -def correlation_type_range_type_json(correlation_type, date_from, date_to): - date_range = [] - if date_from is not None and date_to is not None: - #change format - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - - range_type = [] - all_types_id = get_all_types_id(correlation_type) - - # one day - if len(date_range) == 1: - for type_id in all_types_id: - day_type = {} - # init 0 - for typ_id in all_types_id: - day_type[typ_id] = 0 - day_type['date'] = type_id - num_day_type_id = 0 - all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date_range[0])) - if all_keys: - for val in all_keys: - num_day_type_id += int(val) - day_type[type_id]= num_day_type_id - - #if day_type[type_id] != 0: - range_type.append(day_type) - - else: - # display type_id - for date in date_range: - day_type = {} - day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] - for type_id in all_types_id: - num_day_type_id = 0 - all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date)) - if all_keys: - for val in all_keys: - num_day_type_id += int(val) - day_type[type_id]= num_day_type_id - range_type.append(day_type) - - return jsonify(range_type) - -# ============= ROUTES ============== - - -############################ PGPDump ############################ - -@hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR -@login_required -@login_read_only -def pgp_by_type_json(): - type_id = request.args.get('type_id') - date_from = request.args.get('date_from') - - if date_from is None: - date_from = datetime.date.today().strftime("%Y%m%d") - - #retrieve + char - type_id = type_id.replace(' ', '+') - default = False - - if type_id is None: - default = True - all_type = ['key', 'name', 'mail'] - else: - all_type = [ type_id ] - - num_day_type = 30 - date_range = get_date_range(num_day_type) - - #verify input - if verify_pgp_type_id(type_id) or default: - - type_value = [] - - range_decoder = [] - for date in date_range: - day_type_id = {} - day_type_id['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] - for type_pgp in all_type: - all_vals_key = r_serv_metadata.hvals('pgp:{}:date'.format(type_id, date)) - num_day_type_id = 0 - if all_vals_key is not None: - for val_key in all_vals_key: - num_day_type_id += int(val_key) - day_type_id[type_pgp]= num_day_type_id - range_decoder.append(day_type_id) - - return jsonify(range_decoder) - else: - return jsonify() - -############################ DateRange ############################ -@hashDecoded.route("/correlation/pgpdump", methods=['GET']) -@login_required -@login_read_only -def pgpdump_page(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - type_id = request.args.get('type_id') - - show_decoded_files = request.args.get('show_decoded_files') - res = main_correlation_page('pgpdump', type_id, date_from, date_to, show_decoded_files) - return res - -@hashDecoded.route("/correlation/cryptocurrency", methods=['GET']) -@login_required -@login_read_only -def cryptocurrency_page(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - type_id = request.args.get('type_id') - - show_decoded_files = request.args.get('show_decoded_files') - res = main_correlation_page('cryptocurrency', type_id, date_from, date_to, show_decoded_files) - return res - -@hashDecoded.route("/correlation/username", methods=['GET']) -@login_required -@login_read_only -def username_page(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - type_id = request.args.get('type_id') - - show_decoded_files = request.args.get('show_decoded_files') - res = main_correlation_page('username', type_id, date_from, date_to, show_decoded_files) - return res - -@hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST']) -@login_required -@login_read_only -def all_pgpdump_search(): - date_from = request.form.get('date_from') - date_to = request.form.get('date_to') - type_id = request.form.get('type') - show_decoded_files = request.form.get('show_decoded_files') - return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) - -@hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST']) -@login_required -@login_read_only -def all_cryptocurrency_search(): - date_from = request.form.get('date_from') - date_to = request.form.get('date_to') - type_id = request.form.get('type') - show_decoded_files = request.form.get('show_decoded_files') - return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) - -@hashDecoded.route("/correlation/all_username_search", methods=['POST']) -@login_required -@login_read_only -def all_username_search(): - date_from = request.form.get('date_from') - date_to = request.form.get('date_to') - type_id = request.form.get('type') - show_decoded_files = request.form.get('show_decoded_files') - return redirect(url_for('hashDecoded.username_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) - - - - - - -@hashDecoded.route('/correlation/cryptocurrency_range_type_json') -@login_required -@login_read_only -def cryptocurrency_range_type_json(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_type_range_type_json('cryptocurrency', date_from, date_to) - -@hashDecoded.route('/correlation/pgpdump_range_type_json') -@login_required -@login_read_only -def pgpdump_range_type_json(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_type_range_type_json('pgpdump', date_from, date_to) - -@hashDecoded.route('/correlation/username_range_type_json') -@login_required -@login_read_only -def username_range_type_json(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_type_range_type_json('username', date_from, date_to) - -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## -########################################################################################## - - -# # TODO: REFRACTOR -@hashDecoded.route('/correlation/pgpdump_graph_line_json') -@login_required -@login_read_only -def pgpdump_graph_line_json(): - type_id = request.args.get('type_id') - key_id = request.args.get('key_id') - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_graph_line_json('pgpdump', type_id, key_id, date_from, date_to) - -def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, date_to): - # verify input - if key_id is not None and ail_objects.is_valid_object_subtype(correlation_type, type_id) and ail_objects.exists_obj(correlation_type, type_id, key_id): - - if date_from is None or date_to is None: - nb_days_seen_in_pastes = 30 - else: - # # TODO: # FIXME: - nb_days_seen_in_pastes = 30 - - date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes) - - json_seen_in_paste = [] - for date in date_range_seen_in_pastes: - nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date), key_id) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] - json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)}) - - return jsonify(json_seen_in_paste) - else: - return jsonify() - -@hashDecoded.route('/correlation/cryptocurrency_graph_line_json') -@login_required -@login_read_only -def cryptocurrency_graph_line_json(): - type_id = request.args.get('type_id') - key_id = request.args.get('key_id') - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_graph_line_json('cryptocurrency', type_id, key_id, date_from, date_to) - -@hashDecoded.route('/correlation/username_graph_line_json') -@login_required -@login_read_only -def username_graph_line_json(): - type_id = request.args.get('type_id') - key_id = request.args.get('key_id') - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - return correlation_graph_line_json('username', type_id, key_id, date_from, date_to) - -# ========= REGISTRATION ========= -app.register_blueprint(hashDecoded, url_prefix=baseUrl) diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html deleted file mode 100644 index 117cf940..00000000 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ /dev/null @@ -1,676 +0,0 @@ - - - - - Decoded - AIL - - - - - - - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'sidebars/sidebar_objects.html' %} - -
- -
-
-
-
-
- -
- -
-
-
Select a date range :
-
-
-
- -
-
-
- -
-
Encoding :
- -
File Type :
- -
- - -
- - -
-
- -
-
-
-
-
-
- - {% if l_64|length != 0 %} - {% if date_from|string == date_to|string %} -

{{ date_from }} Decoded files:

- {% else %} -

{{ date_from }} to {{ date_to }} Decoded files:

- {% endif %} - - - - - - - - - - - - - - - {% for b64 in l_64 %} - - - - - - - - - - - {% endfor %} - -
estimated typehashfirst seenlast seennb itemsizeVirus TotalSparkline
  {{ b64[1] }}{{ b64[2] }}{{ b64[5] }}{{ b64[6] }}{{ b64[3] }}{{ b64[4] }} - {% if vt_enabled %} - {% if not b64[7] %} - - - - {% else %} -  VT Report - {% endif %} - - {% else %} - Virus Total submission is disabled - {% endif %} - -
- {% else %} - {% if show_decoded_files %} - {% if date_from|string == date_to|string %} -

{{ date_from }}, No Hashes

- {% else %} -

{{ date_from }} to {{ date_to }}, No Hashes

- {% endif %} - {% endif %} - {% endif %} -
- -
-
- - - - - - - - - - - - - - - - - - - - diff --git a/var/www/modules/hashDecoded/templates/header_hashDecoded.html b/var/www/modules/hashDecoded/templates/header_hashDecoded.html deleted file mode 100644 index 17515c9e..00000000 --- a/var/www/modules/hashDecoded/templates/header_hashDecoded.html +++ /dev/null @@ -1 +0,0 @@ -
  • hashesDecoded
    • diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 6bb4c9fa..24be9d8a 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -265,10 +265,8 @@ $(document).ready(function(){ $("#page-Decoded").addClass("active"); all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"); - {% if dict_object["object_type"] == "pgp" %} - all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.pgpdump_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}"); - {% elif dict_object["object_type"] == "cryptocurrency" %} - all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.cryptocurrency_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}"); + {% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %} + all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}"); {% elif dict_object["object_type"] == "decoded" %} all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_decoded.graphline_json') }}?id={{dict_object["correlation_id"]}}"); {% elif dict_object["object_type"] == "cve" %} diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html deleted file mode 100644 index 6130dd9a..00000000 --- a/var/www/templates/decoded/menu_sidebar.html +++ /dev/null @@ -1,60 +0,0 @@ -
    - - - -
    diff --git a/var/www/modules/hashDecoded/templates/DaysCorrelation.html b/var/www/templates/objects/subtypes_objs_dashboard.html similarity index 78% rename from var/www/modules/hashDecoded/templates/DaysCorrelation.html rename to var/www/templates/objects/subtypes_objs_dashboard.html index a2c84607..78eb1aad 100644 --- a/var/www/modules/hashDecoded/templates/DaysCorrelation.html +++ b/var/www/templates/objects/subtypes_objs_dashboard.html @@ -17,9 +17,9 @@ - - - + + +