AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-26 15:57:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [subtype objects] migrate UI cryptocurrency, pgp, username

Browse source
This commit is contained in:
Terrtia 2023-02-28 11:01:27 +01:00
parent 0fa27c6a51
commit ae6f8af09f
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
18 changed files with 332 additions and 1364 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/lib/ail_core.py
View file

@ -25,6 +25,9 @@ def get_ail_uuid():
def get_all_objects(): def get_all_objects():
return AIL_OBJECTS return AIL_OBJECTS
def get_objects_with_subtypes():
return ['cryptocurrency', 'pgp', 'username']
def get_object_all_subtypes(obj_type): def get_object_all_subtypes(obj_type):
if obj_type == 'cryptocurrency': if obj_type == 'cryptocurrency':
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']

2
bin/lib/objects/CryptoCurrencies.py
View file

@ -120,7 +120,7 @@ class CryptoCurrency(AbstractSubtypeObject):
return obj return obj
def get_meta(self, options=set()): def get_meta(self, options=set()):
meta = self._get_meta() meta = self._get_meta(options=options)
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True) meta['tags'] = self.get_tags(r_list=True)

2
bin/lib/objects/Pgps.py
View file

@ -43,7 +43,7 @@ class Pgp(AbstractSubtypeObject):
# # TODO: # # TODO:
def get_meta(self, options=set()): def get_meta(self, options=set()):
meta = self._get_meta() meta = self._get_meta(options=options)
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True) meta['tags'] = self.get_tags(r_list=True)

2
bin/lib/objects/Usernames.py
View file

@ -64,7 +64,7 @@ class Username(AbstractSubtypeObject):
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
def get_meta(self, options=set()): def get_meta(self, options=set()):
meta = self._get_meta() meta = self._get_meta(options=options)
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True) meta['tags'] = self.get_tags(r_list=True)

82
bin/lib/objects/abstract_subtype_object.py
View file

@ -8,6 +8,7 @@ Base Class for AIL Objects
################################## ##################################
import os import os
import sys import sys
from abc import ABC
# from flask import url_for # from flask import url_for
@ -16,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.objects.abstract_object import AbstractObject from lib.objects.abstract_object import AbstractObject
from lib.ail_core import get_object_all_subtypes
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.item_basic import is_crawled, get_item_domain from lib.item_basic import is_crawled, get_item_domain
from lib.data_retention_engine import update_obj_date from lib.data_retention_engine import update_obj_date
@ -31,7 +33,7 @@ config_loader = None
# # FIXME: SAVE SUBTYPE NAMES ????? # # FIXME: SAVE SUBTYPE NAMES ?????
class AbstractSubtypeObject(AbstractObject): class AbstractSubtypeObject(AbstractObject, ABC):
""" """
Abstract Subtype Object Abstract Subtype Object
""" """
@ -80,11 +82,19 @@ class AbstractSubtypeObject(AbstractObject):
else: else:
return int(nb) return int(nb)
def _get_meta(self): def _get_meta(self, options=None):
meta_dict = {'first_seen': self.get_first_seen(), if options is None:
'last_seen': self.get_last_seen(), options = set()
'nb_seen': self.get_nb_seen()} meta = {'first_seen': self.get_first_seen(),
return meta_dict 'last_seen': self.get_last_seen(),
'nb_seen': self.get_nb_seen()}
if 'icon' in options:
meta['icon'] = self.get_svg_icon()
if 'link' in options:
meta['link'] = self.get_link()
if 'sparkline' in options:
meta['sparkline'] = self.get_sparkline()
return meta
def set_first_seen(self, first_seen): def set_first_seen(self, first_seen):
r_object.hset(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen', first_seen) r_object.hset(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen', first_seen)
@ -111,6 +121,17 @@ class AbstractSubtypeObject(AbstractObject):
for date in Date.get_previous_date_list(6): for date in Date.get_previous_date_list(6):
sparkline.append(self.get_nb_seen_by_date(date)) sparkline.append(self.get_nb_seen_by_date(date))
return sparkline return sparkline
def get_graphline(self, date_from=None, date_to=None):
graphline = []
# TODO get by daterange
# if date_from and date_to:
dates = Date.get_date_range(30)
for date in dates:
nb = self.get_nb_seen_by_date(date)
date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}'
graphline.append({'date': date, 'value': nb})
return graphline
# #
# HANDLE Others objects ???? # HANDLE Others objects ????
# #
@ -151,3 +172,52 @@ class AbstractSubtypeObject(AbstractObject):
def get_all_id(obj_type, subtype): def get_all_id(obj_type, subtype):
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1) return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
def get_subtypes_objs_by_date(obj_type, subtype, date):
return r_object.hkeys(f'{obj_type}:{subtype}:{date}')
def get_subtypes_objs_by_daterange(obj_type, date_from, date_to, subtype=None):
if subtype:
subtypes = [subtype]
else:
subtypes = get_object_all_subtypes(obj_type)
objs = set()
for date in Date.get_daterange(date_from, date_to):
for subtype in subtypes:
for obj_id in get_subtypes_objs_by_date(obj_type, subtype, date):
objs.add((obj_type, subtype, obj_id))
return objs
def get_subtypes_objs_range_json(obj_type, date_from, date_to):
objs_range = []
dates = Date.get_daterange(date_from, date_to)
if len(dates) == 1:
dict_subtype = {}
subtypes = get_object_all_subtypes(obj_type)
for subtype in subtypes:
dict_subtype[subtype] = 0
for subtype in get_object_all_subtypes(obj_type):
day_dict = dict_subtype.copy()
day_dict['date'] = subtype
# if don't filter duplicates
# nb = 0
# for val in r_object.hvals(f'{obj_type}:{subtype}:{dates[0]}'):
# nb += int(val)
# day_dict[subtype] = nb
day_dict[subtype] = r_object.hlen(f'{obj_type}:{subtype}:{dates[0]}')
objs_range.append(day_dict)
else:
subtypes = get_object_all_subtypes(obj_type)
for date in dates:
day_dict = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'}
for subtype in subtypes:
# if don't filter duplicates
# nb = 0
# for val in r_object.hvals(f'{obj_type}:{subtype}:{date}'):
# nb += int(val)
# day_dict[subtype] = nb
day_dict[subtype] = r_object.hlen(f'{obj_type}:{subtype}:{date}')
objs_range.append(day_dict)
return objs_range

2
bin/lib/objects/ail_objects.py
View file

@ -130,7 +130,7 @@ def add_obj_tags(obj_type, subtype, id, tags):
# -TAGS- # # -TAGS- #
def get_object_meta(obj_type, subtype, id, options=[], flask_context=False): def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
obj = get_object(obj_type, subtype, id) obj = get_object(obj_type, subtype, id)
meta = obj.get_meta(options=options) meta = obj.get_meta(options=options)
meta['icon'] = obj.get_svg_icon() meta['icon'] = obj.get_svg_icon()

10
bin/packages/Date.py
View file

@ -129,6 +129,16 @@ def substract_date(date_from, date_to):
l_date.append( date.strftime('%Y%m%d') ) l_date.append( date.strftime('%Y%m%d') )
return l_date return l_date
def get_daterange(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append(date.strftime('%Y%m%d'))
return l_date
def validate_str_date(str_date, separator=''): def validate_str_date(str_date, separator=''):
try: try:
datetime.datetime.strptime(str_date, '%Y{}%m{}%d'.format(separator, separator)) datetime.datetime.strptime(str_date, '%Y{}%m{}%d'.format(separator, separator))

2
var/www/Flask_server.py
View file

@ -48,6 +48,7 @@ from blueprints.ail_2_ail_sync import ail_2_ail_sync
from blueprints.settings_b import settings_b from blueprints.settings_b import settings_b
from blueprints.objects_cve import objects_cve from blueprints.objects_cve import objects_cve
from blueprints.objects_decoded import objects_decoded from blueprints.objects_decoded import objects_decoded
from blueprints.objects_subtypes import objects_subtypes
Flask_dir = os.environ['AIL_FLASK'] Flask_dir = os.environ['AIL_FLASK']
@ -109,6 +110,7 @@ app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
app.register_blueprint(settings_b, url_prefix=baseUrl) app.register_blueprint(settings_b, url_prefix=baseUrl)
app.register_blueprint(objects_cve, url_prefix=baseUrl) app.register_blueprint(objects_cve, url_prefix=baseUrl)
app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_decoded, url_prefix=baseUrl)
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
# ========= =========# # ========= =========#
# ========= Cookie name ======== # ========= Cookie name ========

10
var/www/blueprints/correlation.py
View file

@ -163,7 +163,7 @@ def get_description():
# # TODO: return error json # # TODO: return error json
if not ail_objects.exists_obj(object_type, type_id, correlation_id): if not ail_objects.exists_obj(object_type, type_id, correlation_id):
return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
# oject exist # object exist
else: else:
res = ail_objects.get_object_meta(object_type, type_id, correlation_id, flask_context=True) res = ail_objects.get_object_meta(object_type, type_id, correlation_id, flask_context=True)
return jsonify(res) return jsonify(res)
@ -187,11 +187,3 @@ def graph_node_json():
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
return jsonify(json_graph) return jsonify(json_graph)
@correlation.route('/correlation/subtype_search', methods=['POST'])
@login_required
@login_read_only
def subtype_search():
obj_type = request.form.get('object_type')
obj_subtype = request.form.get('object_subtype')
obj_id = request.form.get('object_id')
return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=obj_subtype, id=obj_id))

166
var/www/blueprints/objects_subtypes.py Normal file
View file

@ -0,0 +1,166 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
from flask_login import login_required, current_user
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_core
from lib.objects import abstract_subtype_object
from lib.objects import ail_objects
from lib.objects import CryptoCurrencies
from packages import Date
# ============ BLUEPRINT ============
objects_subtypes = Blueprint('objects_subtypes', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
# ============ VARIABLES ============
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============ FUNCTIONS ============
# TODO VERIFY SUBTYPE
def subtypes_objects_dashboard(obj_type, f_request):
if request.method == 'POST':
date_from = f_request.form.get('from')
date_to = f_request.form.get('to')
subtype = f_request.form.get('subtype')
show_objects = bool(f_request.form.get('show_objects'))
endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{obj_type}')
endpoint_dashboard = f'{endpoint_dashboard}?from={date_from}&to={date_to}'
if subtype:
if subtype == 'All types':
subtype = None
if subtype:
if not ail_objects.is_valid_object_subtype(obj_type, subtype):
subtype = None
if subtype:
endpoint_dashboard = f'{endpoint_dashboard}&subtype={subtype}'
if show_objects:
endpoint_dashboard = f'{endpoint_dashboard}&show_objects={show_objects}'
return redirect(endpoint_dashboard)
else:
date_from = f_request.args.get('from')
date_to = f_request.args.get('to')
subtype = f_request.args.get('subtype')
show_objects = bool(f_request.args.get('show_objects'))
# Date
date = Date.sanitise_date_range(date_from, date_to)
date_from = date['date_from']
date_to = date['date_to']
daily_type_chart = date_from == date_to
# Subtype
if subtype == 'All types':
subtype = None
if subtype:
if not ail_objects.is_valid_object_subtype(obj_type, subtype):
subtype = None
objs = []
if show_objects:
subtypes_objs = abstract_subtype_object.get_subtypes_objs_by_daterange(obj_type, date_from, date_to,
subtype=subtype)
if subtypes_objs:
for obj_t, obj_subtype, obj_id in subtypes_objs:
objs.append(ail_objects.get_object_meta(obj_t, obj_subtype, obj_id, options={'sparkline'}))
endpoint_dashboard = f'objects_subtypes.objects_dashboard_{obj_type}'
return render_template('subtypes_objs_dashboard.html', date_from=date_from, date_to=date_to,
daily_type_chart = daily_type_chart, show_objects=show_objects,
obj_type=obj_type, subtype=subtype, objs=objs,
subtypes = ail_core.get_object_all_subtypes(obj_type),
endpoint_dashboard=endpoint_dashboard)
# ============= ROUTES ==============
@objects_subtypes.route("/objects/cryptocurrencies", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_cryptocurrency():
return subtypes_objects_dashboard('cryptocurrency', request)
@objects_subtypes.route("/objects/pgps", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_pgp():
return subtypes_objects_dashboard('pgp', request)
@objects_subtypes.route("/objects/usernames", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_username():
return subtypes_objects_dashboard('username', request)
# TODO REDIRECT
@objects_subtypes.route("/objects/subtypes/post", methods=['POST'])
@login_required
@login_read_only
def objects_subtypes_dashboard_post():
obj_type = request.form.get('obj_type')
if obj_type not in ail_core.get_objects_with_subtypes():
return create_json_response({'error': 'Invalid Object type'}, 400)
return subtypes_objects_dashboard(obj_type, request)
@objects_subtypes.route("/objects/subtypes/range/json", methods=['GET'])
@login_required
@login_read_only
def objects_subtypes_range_json():
obj_type = request.args.get('type')
if obj_type not in ail_core.get_objects_with_subtypes():
return create_json_response({'error': 'Invalid Object type'}, 400)
date_from = request.args.get('from')
date_to = request.args.get('to')
date = Date.sanitise_date_range(date_from, date_to)
date_from = date['date_from']
date_to = date['date_to']
return jsonify(abstract_subtype_object.get_subtypes_objs_range_json(obj_type, date_from, date_to))
@objects_subtypes.route("/objects/subtypes/search", methods=['POST'])
@login_required
@login_read_only
def objects_subtypes_search():
obj_type = request.form.get('type')
subtype = request.form.get('subtype')
obj_id = request.form.get('id')
if obj_type not in ail_core.get_objects_with_subtypes():
return create_json_response({'error': 'Invalid Object type'}, 400)
obj = ail_objects.get_object(obj_type, subtype, obj_id)
if not obj.exists():
abort(404)
else:
# TODO Search object
return redirect(obj.get_link(flask_context=True))
@objects_subtypes.route("/objects/subtypes/graphline/json", methods=['GET'])
@login_required
@login_read_only
def objects_cve_graphline_json():
obj_type = request.args.get('type')
subtype = request.args.get('subtype')
obj_id = request.args.get('id')
if obj_type not in ail_core.get_objects_with_subtypes():
return create_json_response({'error': 'Invalid Object type'}, 400)
obj = ail_objects.get_object(obj_type, subtype, obj_id)
if not obj.exists():
abort(404)
else:
return jsonify(obj.get_graphline())

5
var/www/modules/Flask_config.py
View file

@ -31,12 +31,11 @@ r_serv_log = config_loader.get_redis_conn("Redis_Log")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # -> TODO MIGRATE /correlation/ subtypes objects
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER
# # # # # # # # # # # # # #
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint
r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint
# Logger (Redis) # Logger (Redis)
redis_logger = publisher redis_logger = publisher

544
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -1,544 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import os
import sys
import datetime
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects import ail_objects
from packages.Date import Date
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata
vt_enabled = Flask_config.vt_enabled
vt_auth = Flask_config.vt_auth
PASTES_FOLDER = Flask_config.PASTES_FOLDER
hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
## TODO: put me in option
all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
all_pgpdump = ['key', 'name', 'mail']
all_username = ['telegram', 'twitter', 'jabber']
# ============ FUNCTIONS ============
def get_date_range(num_day):
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = []
for i in range(0, num_day+1):
date_list.append(date.substract_day(i))
return list(reversed(date_list))
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
def get_icon(correlation_type, type_id):
icon_text = 'fas fa-sticky-note'
if correlation_type == 'pgpdump':
# set type_id icon
if type_id == 'key':
icon_text = 'fas fa-key'
elif type_id == 'name':
icon_text = 'fas fa-user-tag'
elif type_id == 'mail':
icon_text = 'fas fa-at'
else:
icon_text = 'times'
elif correlation_type == 'cryptocurrency':
if type_id == 'bitcoin':
icon_text = 'fab fa-btc'
elif type_id == 'monero':
icon_text = 'fab fa-monero'
elif type_id == 'ethereum':
icon_text = 'fab fa-ethereum'
else:
icon_text = 'fas fa-coins'
elif correlation_type == 'username':
if type_id == 'telegram':
icon_text = 'fab fa-telegram-plane'
elif type_id == 'twitter':
icon_text = 'fab fa-twitter'
elif type_id == 'jabber':
icon_text = 'fas fa-user'
return icon_text
def get_icon_text(correlation_type, type_id):
icon_text = '\uf249'
if correlation_type == 'pgpdump':
if type_id == 'key':
icon_text = '\uf084'
elif type_id == 'name':
icon_text = '\uf507'
elif type_id == 'mail':
icon_text = '\uf1fa'
else:
icon_text = 'times'
elif correlation_type == 'cryptocurrency':
if type_id == 'bitcoin':
icon_text = '\uf15a'
elif type_id == 'monero':
icon_text = '\uf3d0'
elif type_id == 'ethereum':
icon_text = '\uf42e'
else:
icon_text = '\uf51e'
elif correlation_type == 'username':
if type_id == 'telegram':
icon_text = '\uf2c6'
elif type_id == 'twitter':
icon_text = '\uf099'
elif type_id == 'jabber':
icon_text = '\uf007'
return icon_text
def get_all_types_id(correlation_type):
if correlation_type == 'pgpdump':
return all_pgpdump
elif correlation_type == 'cryptocurrency':
return all_cryptocurrency
elif correlation_type == 'username':
return all_username
else:
return []
def get_key_id_metadata(obj_type, subtype, obj_id):
obj = ail_objects.get_object_meta(obj_type, subtype, obj_id)
return obj
def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date_day), key_id)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_correlation_type_search_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.all_pgpdump_search'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.all_cryptocurrency_search'
elif correlation_type == 'username':
endpoint = 'hashDecoded.all_username_search'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_correlation_type_page_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.pgpdump_page'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.cryptocurrency_page'
elif correlation_type == 'username':
endpoint = 'hashDecoded.username_page'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_show_key_id_endpoint(correlation_type):
return 'correlation.show_correlation'
def get_range_type_json_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.pgpdump_range_type_json'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.cryptocurrency_range_type_json'
elif correlation_type == 'username':
endpoint = 'hashDecoded.username_range_type_json'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
############ CORE CORRELATION ############
def main_correlation_page(correlation_type, type_id, date_from, date_to, show_decoded_files):
if type_id == 'All types':
type_id = None
# verify type input
if type_id is not None:
#retrieve char
type_id = type_id.replace(' ', '')
if not ail_objects.is_valid_object_subtype(correlation_type, type_id):
type_id = None
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
# display day type bar chart
if len(date_range) == 1 and type is None:
daily_type_chart = True
daily_date = date_range[0]
else:
daily_type_chart = False
daily_date = None
if type_id is None:
all_type_id = get_all_types_id(correlation_type)
else:
all_type_id = type_id
l_keys_id_dump = set()
if show_decoded_files:
for date in date_range:
if isinstance(all_type_id, str):
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, all_type_id, date))
if l_dump:
for dump in l_dump:
l_keys_id_dump.add( (dump, all_type_id) )
else:
for typ_id in all_type_id:
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, typ_id, date))
if l_dump:
for dump in l_dump:
l_keys_id_dump.add( (dump, typ_id) )
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_id = 0
keys_id_metadata = {}
for dump_res in l_keys_id_dump:
new_key_id, typ_id = dump_res
keys_id_metadata[new_key_id] = get_key_id_metadata(correlation_type, typ_id, new_key_id)
if keys_id_metadata[new_key_id]:
keys_id_metadata[new_key_id]['type_id'] = typ_id
keys_id_metadata[new_key_id]['type_icon'] = get_icon(correlation_type, typ_id)
keys_id_metadata[new_key_id]['sparklines_data'] = list_sparkline_type_id_values(date_range_sparkline, correlation_type, typ_id, new_key_id)
keys_id_metadata[new_key_id]['sparklines_id'] = sparkline_id
sparkline_id += 1
l_type = get_all_types_id(correlation_type)
correlation_type_n = correlation_type
if correlation_type_n=='pgpdump':
correlation_type_n = 'pgp'
return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata,
correlation_type=correlation_type,
correlation_type_n=correlation_type_n,
correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type),
correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type),
show_key_id_endpoint=get_show_key_id_endpoint(correlation_type),
range_type_json_endpoint=get_range_type_json_endpoint(correlation_type),
l_type=l_type, type_id=type_id,
daily_type_chart=daily_type_chart, daily_date=daily_date,
date_from=date_from, date_to=date_to,
show_decoded_files=show_decoded_files)
def correlation_type_range_type_json(correlation_type, date_from, date_to):
date_range = []
if date_from is not None and date_to is not None:
#change format
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
range_type = []
all_types_id = get_all_types_id(correlation_type)
# one day
if len(date_range) == 1:
for type_id in all_types_id:
day_type = {}
# init 0
for typ_id in all_types_id:
day_type[typ_id] = 0
day_type['date'] = type_id
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date_range[0]))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
#if day_type[type_id] != 0:
range_type.append(day_type)
else:
# display type_id
for date in date_range:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type_id in all_types_id:
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
range_type.append(day_type)
return jsonify(range_type)
# ============= ROUTES ==============
############################ PGPDump ############################
@hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR
@login_required
@login_read_only
def pgp_by_type_json():
type_id = request.args.get('type_id')
date_from = request.args.get('date_from')
if date_from is None:
date_from = datetime.date.today().strftime("%Y%m%d")
#retrieve + char
type_id = type_id.replace(' ', '+')
default = False
if type_id is None:
default = True
all_type = ['key', 'name', 'mail']
else:
all_type = [ type_id ]
num_day_type = 30
date_range = get_date_range(num_day_type)
#verify input
if verify_pgp_type_id(type_id) or default:
type_value = []
range_decoder = []
for date in date_range:
day_type_id = {}
day_type_id['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type_pgp in all_type:
all_vals_key = r_serv_metadata.hvals('pgp:{}:date'.format(type_id, date))
num_day_type_id = 0
if all_vals_key is not None:
for val_key in all_vals_key:
num_day_type_id += int(val_key)
day_type_id[type_pgp]= num_day_type_id
range_decoder.append(day_type_id)
return jsonify(range_decoder)
else:
return jsonify()
############################ DateRange ############################
@hashDecoded.route("/correlation/pgpdump", methods=['GET'])
@login_required
@login_read_only
def pgpdump_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
show_decoded_files = request.args.get('show_decoded_files')
res = main_correlation_page('pgpdump', type_id, date_from, date_to, show_decoded_files)
return res
@hashDecoded.route("/correlation/cryptocurrency", methods=['GET'])
@login_required
@login_read_only
def cryptocurrency_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
show_decoded_files = request.args.get('show_decoded_files')
res = main_correlation_page('cryptocurrency', type_id, date_from, date_to, show_decoded_files)
return res
@hashDecoded.route("/correlation/username", methods=['GET'])
@login_required
@login_read_only
def username_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
show_decoded_files = request.args.get('show_decoded_files')
res = main_correlation_page('username', type_id, date_from, date_to, show_decoded_files)
return res
@hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST'])
@login_required
@login_read_only
def all_pgpdump_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST'])
@login_required
@login_read_only
def all_cryptocurrency_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route("/correlation/all_username_search", methods=['POST'])
@login_required
@login_read_only
def all_username_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.username_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route('/correlation/cryptocurrency_range_type_json')
@login_required
@login_read_only
def cryptocurrency_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_type_range_type_json('cryptocurrency', date_from, date_to)
@hashDecoded.route('/correlation/pgpdump_range_type_json')
@login_required
@login_read_only
def pgpdump_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_type_range_type_json('pgpdump', date_from, date_to)
@hashDecoded.route('/correlation/username_range_type_json')
@login_required
@login_read_only
def username_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_type_range_type_json('username', date_from, date_to)
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
# # TODO: REFRACTOR
@hashDecoded.route('/correlation/pgpdump_graph_line_json')
@login_required
@login_read_only
def pgpdump_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_graph_line_json('pgpdump', type_id, key_id, date_from, date_to)
def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, date_to):
# verify input
if key_id is not None and ail_objects.is_valid_object_subtype(correlation_type, type_id) and ail_objects.exists_obj(correlation_type, type_id, key_id):
if date_from is None or date_to is None:
nb_days_seen_in_pastes = 30
else:
# # TODO: # FIXME:
nb_days_seen_in_pastes = 30
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
json_seen_in_paste = []
for date in date_range_seen_in_pastes:
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date), key_id)
if nb_seen_this_day is None:
nb_seen_this_day = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
return jsonify(json_seen_in_paste)
else:
return jsonify()
@hashDecoded.route('/correlation/cryptocurrency_graph_line_json')
@login_required
@login_read_only
def cryptocurrency_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_graph_line_json('cryptocurrency', type_id, key_id, date_from, date_to)
@hashDecoded.route('/correlation/username_graph_line_json')
@login_required
@login_read_only
def username_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_graph_line_json('username', type_id, key_id, date_from, date_to)
# ========= REGISTRATION =========
app.register_blueprint(hashDecoded, url_prefix=baseUrl)

676
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -1,676 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Decoded - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
.input-group .form-control {
position: unset;
}
.line {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.bar {
fill: steelblue;
}
.bar:hover{
fill: brown;
cursor: pointer;
}
.bar_stack:hover{
cursor: pointer;
}
.pie_path:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="row">
<div class="col-xl-10">
<div class="mt-1" id="barchart_type">
</div>
</div>
<div class="col-xl-2">
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
<div class="card-body text-center py-2">
<h6 class="card-title" style="color:#286090;">Select a date range :</h5>
<form action="{{ url_for('hashDecoded.all_hash_search') }}" id="hash_selector_form" method='post'>
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
</div>
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
</div>
<div class="mt-1" style="font-size: 14px;color:#286090;">Encoding :</div>
<select class="custom-select" name="encoding">
<option>All encoding</option>
{% for encod in all_encoding %}
{% if encoding|string() == encod|string() %}
<option selected>{{ encod }}</option>
{% else %}
<option>{{ encod }}</option>
{% endif %}
{% endfor %}
</select>
<div class="mt-1" style="font-size: 14px;color:#286090;">File Type :</div>
<select class="custom-select" name="type">
<option>All types</option>
{% for typ in l_type %}
{% if type|string() == typ|string() %}
<option selected>{{ typ }}</option>
{% else %}
<option>{{ typ }}</option>
{% endif %}
{% endfor %}
</select>
<div class="form-check my-1">
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
<label class="form-check-label" for="checkbox-input-show">
<div style="color:#286090; font-size: 14px;">
Show decoded files <i class="fas fa-file"></i>
</div>
</label>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fas fa-copy"></i> Search
</button>
<form>
</div>
</div>
<div id="pie_chart_encoded">
</div>
<div id="pie_chart_top5_types">
</div>
</div>
</div>
{% if l_64|length != 0 %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }} Decoded files: </h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }} Decoded files: </h3>
{% endif %}
<table id="tableb64" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>estimated type</th>
<th>hash</th>
<th>first seen</th>
<th>last seen</th>
<th>nb item</th>
<th>size</th>
<th>Virus Total</th>
<th>Sparkline</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for b64 in l_64 %}
<tr>
<td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=decoded&id={{ b64[2] }}">{{ b64[2] }}</a></td>
<td>{{ b64[5] }}</td>
<td>{{ b64[6] }}</td>
<td>{{ b64[3] }}</td>
<td>{{ b64[4] }}</td>
<td>
{% if vt_enabled %}
{% if not b64[7] %}
<darkbutton_{{ b64[2] }}>
<button id="submit_vt_{{ b64[2] }}" class="btn btn-secondary" style="font-size: 14px;" onclick="sendFileToVT('{{ b64[2] }}')">
<i class="fas fa-paper-plane"></i>&nbsp;Send this file to VT
</button>
</darkbutton_{{ b64[2] }}>
{% else %}
<a class="btn btn-secondary" target="_blank" href="{{ b64[8] }}" style="font-size: 14px;"><i class="fas fa-link"></i>&nbsp;VT Report</a>
{% endif %}
<button class="btn btn-outline-dark" onclick="updateVTReport('{{ b64[2] }}')" style="font-size: 14px;">
<div id="report_vt_{{ b64[2] }}"><i class="fas fa-sync-alt"></i>&nbsp;{{ b64[9] }}</div>
</button>
{% else %}
Virus Total submission is disabled
{% endif %}
</td>
<td id="sparklines_{{ b64[2] }}" style="text-align:center;"></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
{% if show_decoded_files %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No Hashes</h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }}, No Hashes</h3>
{% endif %}
{% endif %}
{% endif %}
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_dashboard").addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#tableb64').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
});
{% if type %}
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.hash_by_type_json') }}?type={{type}}", 'id');
{% elif daily_type_chart %}
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
{% else %}
chart.stackBarChart = barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
{% endif %}
draw_pie_chart("pie_chart_encoded" ,"{{ url_for('objects_decoded.decoder_pie_chart_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
chart.onResize();
$(window).on("resize", function() {
chart.onResize();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function updateVTReport(hash) {
//updateReport
$.getJSON("{{ url_for('objects_decoded.refresh_vt_report') }}?id="+hash,
function(data) {
content = '<i class="fas fa-sync-alt"></i> ' +data['report']
$( "#report_vt_"+hash ).html(content);
});
}
function sendFileToVT(hash) {
//send file to vt
$.getJSON("{{ url_for('objects_decoded.send_to_vt') }}?id="+hash,
function(data) {
var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
$('#submit_vt_'+hash).remove();
$('darkbutton_'+hash).append(content);
});
}
</script>
<script>
{% for b64 in l_64 %}
sparkline("sparklines_{{ b64[2] }}", {{ b64[10] }}, {});
{% endfor %}
</script>
<script>
var margin = {top: 20, right: 100, bottom: 55, left: 45},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type_stack(url, id) {
d3.json(url)
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("class", "bar")
{% if date_from|string == date_to|string and type is none %}
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type="+d })
.attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-18)" )
{% else %}
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-40)" )
{% endif %}
.style("text-anchor", "end");
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and type is none %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type='+d.label+'&encoding='+d.name; });
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
{% else %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
{% endif %}
data.forEach(function(d) {
if(d.total != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total))
{% if date_from|string == date_to|string and type is none %}
.on("click", function () {window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type='+d.date })
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function () {window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}?type={{type}}&date_from="+d.date+'&date_to='+d.date })
{% else %}
.on("click", function () {window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d.date+'&date_to='+d.date })
{% endif %}
.style("text-anchor", "middle")
.text(d.total);
}
});
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: "<b><span id='tooltip-id-name-bar'></span></b>",
placement: 'top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "<span id='tooltip-id-label'></span>" +
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
});
$(this).popover('show');
$("#tooltip-id-name-bar").text(d.name);
$("#tooltip-id-label").text(d.label);
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
}
chart.onResize = function () {
var aspect = 1000 / 500, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.chart = chart;
</script>
<script>
function draw_pie_chart(id, url_json, pie_on_click_url) {
var width_pie = 200;
var height_pie = 200;
var padding_pie = 10;
var opacity_pie = .8;
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
var div_pie = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var svg_pie = d3.select("#"+id)
.append('svg')
.attr("width", '100%')
.attr("height", '100%')
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
.attr('preserveAspectRatio','xMinYMin')
var g_pie = svg_pie.append('g')
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
var arc_pie = d3.arc()
.innerRadius(0)
.outerRadius(radius_pie);
d3.json(url_json)
.then(function(data){
var pie_pie = d3.pie()
.value(function(d) { return d.value; })
.sort(null);
var path_pie = g_pie.selectAll('path')
.data(pie_pie(data))
.enter()
.append("g")
.append('path')
.attr('d', arc_pie)
.attr('fill', (d,i) => color_pie(i))
.attr('class', 'pie_path')
.on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
.style('opacity', opacity_pie)
.style('stroke', 'white');
});
function mouseovered_pie(d) {
//remove old content
$("#tooltip-id-name").remove();
$("#tooltip-id-value").remove();
// tooltip
var content;
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
div_pie.transition()
.duration(200)
.style("opacity", .9);
div_pie.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.data.name);
$("#tooltip-id-value").text(d.data.value);
}
function mouseouted_pie() {
div_pie.transition()
.duration(500)
.style("opacity", 0);
}
}
</script>
<script>
function barchart_type(url, id) {
var margin = {top: 20, right: 20, bottom: 70, left: 40};
var width = 960 - margin.left - margin.right;
var height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x)
//.tickFormat(d3.time.format("%Y-%m"));
var yAxis = d3.axisLeft(y)
.ticks(10);
/*var svg = d3.select(id).append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.attr("id", "thesvg")
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");*/
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.value = +d.value;
});
x.domain(data.map(function(d) { return d.date; }));
y.domain([0, d3.max(data, function(d) { return d.value; })]);
var label = svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", "-.55em")
{% if daily_type_chart %}
.attr("transform", "rotate(-20)" );
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d });
{% endif %}
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Value ($)");
var bar = svg.selectAll("bar")
.data(data)
.enter().append("rect")
.attr("class", "bar")
//.style("fill", "steelblue")
.attr("x", function(d) { return x(d.date); })
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
{% if type %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
{% endif %}
{% if daily_type_chart %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
{% endif %}
data.forEach(function(d) {
if(d.value != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
//.text(function(d) { return d.value; });
.text(d.value)
.style("text-anchor", "middle")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.value));
}
});
});
}
</script>
</body>
</html>

1
var/www/modules/hashDecoded/templates/header_hashDecoded.html
View file

@ -1 +0,0 @@
<li id='page-hashDecoded'><a href="{{ url_for('objects_decoded.decodeds_dashboard') }}"><i class="fa fa-files-o"></i> hashesDecoded </a></li>

6
var/www/templates/correlation/show_correlation.html
View file

@ -265,10 +265,8 @@ $(document).ready(function(){
$("#page-Decoded").addClass("active"); $("#page-Decoded").addClass("active");
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"); all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}");
{% if dict_object["object_type"] == "pgp" %} {% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.pgpdump_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}"); all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "cryptocurrency" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.cryptocurrency_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "decoded" %} {% elif dict_object["object_type"] == "decoded" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_decoded.graphline_json') }}?id={{dict_object["correlation_id"]}}"); all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_decoded.graphline_json') }}?id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "cve" %} {% elif dict_object["object_type"] == "cve" %}

60
var/www/templates/decoded/menu_sidebar.html
View file

@ -1,60 +0,0 @@
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
<i class="fas fa-align-left"></i>
<span>Toggle Sidebar</span>
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
<h5 class="d-flex text-muted w-100">
<span>Objects</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
<i class="fas fa-lock-open"></i>
<span>Decoded</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
<i class="fas fa-key"></i>
<span>PGP Dumps</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
<i class="fas fa-coins"></i>
<span>Cryptocurrency</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.username_page')}}" id="nav_dashboard_username">
<i class="fas fa-user"></i>
<span>Username</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100">
<span>
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
Format
</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
<li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">
<b>Import</b>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
<b>Export</b>
</a>
</li>
</ul>
</nav>
</div>

117
var/www/modules/hashDecoded/templates/DaysCorrelation.html → var/www/templates/objects/subtypes_objs_dashboard.html
View file

@ -17,9 +17,9 @@
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script> <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script> <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script> <script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script> <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style> <style>
@ -78,20 +78,20 @@
<div class="card border-secondary my-2"> <div class="card border-secondary my-2">
<div class="card-body text-dark"> <div class="card-body text-dark">
<h5 class="card-title">Search {{correlation_type}} by name:</h5> <h5 class="card-title">Search {{obj_type}} by name:</h5>
<form action="{{ url_for('correlation.subtype_search') }}" id="search_subtype_onj" method='post'> <form action="{{ url_for('objects_subtypes.objects_subtypes_search') }}" id="search_subtype_onj" method='post'>
<div class="input-group mb-1"> <div class="input-group mb-1">
<input type="text" class="form-control" name="object_type" value="{{correlation_type}}" hidden> <input type="text" class="form-control" name="type" value="{{obj_type}}" hidden>
<select class="custom-select col-2" name="object_subtype" value="{{obj_type}}" required> <select class="custom-select col-2" name="subtype" value="{{subtype}}" required>
<option value="">{{correlation_type}} Type...</option> <option value="">{{obj_type}} Type...</option>
{% for typ in l_type %} {% for typ in subtypes %}
<option value="{{typ}}">{{typ}}</option> <option value="{{typ}}">{{typ}}</option>
{% endfor %} {% endfor %}
</select> </select>
<input type="text" class="form-control col-8" name="object_id" value="" placeholder="{{correlation_type}} ID" required> <input type="text" class="form-control col-8" name="id" value="" placeholder="{{obj_type}} ID" required>
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button> <button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
</div> </div>
@ -108,20 +108,21 @@
<div class="card mb-3 mt-2" style="background-color:#d9edf7;"> <div class="card mb-3 mt-2" style="background-color:#d9edf7;">
<div class="card-body text-center py-2"> <div class="card-body text-center py-2">
<h6 class="card-title" style="color:#286090;">Select a date range :</h5> <h6 class="card-title" style="color:#286090;">Select a date range :</h5>
<form action="{{ url_for(correlation_type_search_endpoint) }}" id="hash_selector_form" method='post'> <form action="{{ url_for('objects_subtypes.objects_subtypes_dashboard_post') }}" method='post'>
<input type="text" id="obj_type" name="obj_type" value="{{ obj_type }}" hidden>
<div class="input-group" id="date-range-from"> <div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off"> <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="from" autocomplete="off">
</div> </div>
<div class="input-group" id="date-range-to"> <div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off"> <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="to" autocomplete="off">
</div> </div>
<div class="mt-1" style="font-size: 14px;color:#286090;">Type ID :</div> <div class="mt-1" style="font-size: 14px;color:#286090;">Type ID :</div>
<select class="custom-select" name="type"> <select class="custom-select" name="subtype">
<option>All types</option> <option>All types</option>
{% for typ in l_type %} {% for typ in subtypes %}
{% if type_id|string() == typ|string() %} {% if subtype|string() == typ|string() %}
<option selected>{{ typ }}</option> <option selected>{{ typ }}</option>
{% else %} {% else %}
<option>{{ typ }}</option> <option>{{ typ }}</option>
@ -129,10 +130,10 @@
{% endfor %} {% endfor %}
</select> </select>
<div class="form-check my-1"> <div class="form-check my-1">
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}> <input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
<label class="form-check-label" for="checkbox-input-show"> <label class="form-check-label" for="checkbox-input-show">
<div style="color:#286090; font-size: 14px;"> <div style="color:#286090; font-size: 14px;">
Show {{correlation_type}} <i class="fas fa-key"></i> Show {{obj_type}} <i class="fas fa-key"></i>
</div> </div>
</label> </label>
</div> </div>
@ -150,42 +151,51 @@
</div> </div>
</div> </div>
{% if all_metadata|length != 0 %} {% if objs|length != 0 %}
{% if date_from|string == date_to|string %} {% if date_from|string == date_to|string %}
<h3> {{ date_from }} {{correlation_type}}: </h3> <h3> {{date_from[0:4]}}-{{date_from[4:6]}}-{{date_from[6:8]}} {{obj_type}}: </h3>
{% else %} {% else %}
<h3> {{ date_from }} to {{ date_to }} {{correlation_type}}: </h3> <h3> {{date_from[0:4]}}-{{date_from[4:6]}}-{{date_from[6:8]}} to {{date_to[0:4]}}-{{date_to[4:6]}}-{{date_to[6:8]}} {{obj_type}}: </h3>
{% endif %} {% endif %}
<table id="tableb64" class="table table-striped table-bordered"> <table id="tableb64" class="table table-striped table-bordered">
<thead class="bg-dark text-white"> <thead class="bg-dark text-white">
<tr> <tr>
<th>type id</th> <th>Subtype</th>
<th>key id</th> <th>Id</th>
<th>first seen</th> <th>First Seen</th>
<th>last seen</th> <th>Last Seen</th>
<th>nb item</th> <th>Nb Seen</th>
<th>Sparkline</th> <th>Sparkline</th>
</tr> </tr>
</thead> </thead>
<tbody style="font-size: 15px;"> <tbody style="font-size: 15px;">
{% for key_id in all_metadata %} {% for meta in objs %}
<tr> <tr>
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td> <td>
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type={{correlation_type_n}}&subtype={{ all_metadata[key_id]['type_id'] }}&id={{ key_id }}">{{ key_id }}</a></td> <svg height="26" width="26">
<td>{{ all_metadata[key_id]['first_seen'] }}</td> <g class="nodes">
<td>{{ all_metadata[key_id]['last_seen'] }}</td> <circle cx="13" cy="13" r="13" fill="{{ meta['icon']['color'] }}"></circle>
<td>{{ all_metadata[key_id]['nb_seen'] }}</td> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta['icon']['style'] }}" font-size="16px">{{ meta['icon']['icon'] }}</text>
<td id="sparklines_{{ all_metadata[key_id]['sparklines_id'] }}" style="text-align:center;"></td> </g>
</svg>
{{ meta['subtype'] }}
</td>
<td><a target="_blank" href="{{ meta['link'] }}">{{ meta['id'] }}</a></td>
<td>{{meta['first_seen'][0:4]}}-{{meta['first_seen'][4:6]}}-{{meta['first_seen'][6:8]}}</td>
<td>{{meta['last_seen'][0:4]}}-{{meta['last_seen'][4:6]}}-{{meta['last_seen'][6:8]}}</td>
<td>{{ meta['nb_seen'] }}</td>
<td id="sparklines_{{ meta['subtype'] + loop.index0|string }}" style="text-align:center;"></td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
{% else %} {% else %}
{% if show_decoded_files %} {% if show_objects %}
{% if date_from|string == date_to|string %} {% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No Dumped Keys</h3> <h3> {{date_from[0:4]}}-{{date_from[4:6]}}-{{date_from[6:8]}}, No Dumped Keys</h3>
{% else %} {% else %}
<h3> {{ date_from }} to {{ date_to }}, No {{correlation_type}}</h3> <h3> {{date_from[0:4]}}-{{date_from[4:6]}}-{{date_from[6:8]}} to {{date_to[0:4]}}-{{date_to[4:6]}}-{{date_to[6:8]}}, No {% if subtype %}{{ subtype }} {% endif %}{{obj_type}}</h3>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endif %} {% endif %}
@ -199,7 +209,7 @@
var chart = {}; var chart = {};
$(document).ready(function(){ $(document).ready(function(){
$("#page-Decoded").addClass("active"); $("#page-Decoded").addClass("active");
$("#nav_dashboard_{{correlation_type}}").addClass("active"); $("#nav_dashboard_{{obj_type}}").addClass("active");
$('#date-range-from').dateRangePicker({ $('#date-range-from').dateRangePicker({
separator : ' to ', separator : ' to ',
@ -241,13 +251,12 @@
"order": [[ 3, "desc" ]] "order": [[ 3, "desc" ]]
}); });
{% if type_id %} {% if subtype %}
//chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&type_id={{type_id}}", 'id'); barchart_type_stack("{{ url_for('objects_subtypes.objects_subtypes_range_json') }}?type={{obj_type}}&subtype={{subtype}}&from={{date_from}}&to={{date_to}}", 'id');
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id={{type_id}}", 'id'); {% elif subtype or daily_type_chart %}
{% elif type_id or daily_type_chart %} barchart_type_stack("{{ url_for('objects_subtypes.objects_subtypes_range_json') }}?type={{obj_type}}&from={{date_from}}&to={{date_from}}", 'id');
chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
{% else %} {% else %}
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id'); barchart_type_stack("{{ url_for('objects_subtypes.objects_subtypes_range_json') }}?type={{obj_type}}&from={{date_from}}&to={{date_to}}", 'id');
{% endif %} {% endif %}
chart.onResize(); chart.onResize();
@ -272,8 +281,8 @@ function toggle_sidebar(){
</script> </script>
<script> <script>
{% for key_id in all_metadata %} {% for meta in objs %}
sparkline("sparklines_{{ all_metadata[key_id]['sparklines_id'] }}", {{ all_metadata[key_id]['sparklines_data'] }}, {}); sparkline("sparklines_{{ meta['subtype'] + loop.index0|string}}", {{ meta['sparkline'] }}, {});
{% endfor %} {% endfor %}
</script> </script>
@ -331,13 +340,13 @@ function barchart_type_stack(url, id) {
.selectAll("text") .selectAll("text")
.attr("class", "bar") .attr("class", "bar")
{% if date_from|string == date_to|string and type is none %} {% if date_from|string == date_to|string and type is none %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d }) .on("click", function (d) { window.location.href = "{{ url_for(endpoint_dashboard) }}?from={{date_from}}&to={{date_to}}&subtype="+d })
.attr("transform", "rotate(-18)" ) .attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and type is not none %} {% elif date_from|string == date_to|string and type is not none %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d }) .on("click", function (d) { window.location.href = "{{ url_for(endpoint_dashboard) }}?from="+d+'&to='+d })
.attr("transform", "rotate(-18)" ) .attr("transform", "rotate(-18)" )
{% else %} {% else %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d }) .on("click", function (d) { window.location.href = "{{ url_for(endpoint_dashboard) }}?from="+d+'&to='+d })
.attr("transform", "rotate(-40)" ) .attr("transform", "rotate(-40)" )
{% endif %} {% endif %}
.style("text-anchor", "end"); .style("text-anchor", "end");
@ -369,11 +378,11 @@ function barchart_type_stack(url, id) {
.on("mouseover", function (d) { showPopover.call(this, d); }) .on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); }) .on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and type is none %} {% if date_from|string == date_to|string and type is none %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; }); .on("click", function(d){ window.location.href = "{{ url_for(endpoint_dashboard) }}" +'?from={{date_from}}&to={{date_to}}&subtype='+d.label+'&encoding='+d.name; });
{% elif date_from|string == date_to|string and type is not none %} {% elif date_from|string == date_to|string and type is not none %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; }); .on("click", function(d){ window.location.href = "{{ url_for(endpoint_dashboard) }}" +'?subtype={{subtype}}&from='+d.label+'&to='+d.label+'&encoding='+d.name; });
{% else %} {% else %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; }); .on("click", function(d){ window.location.href = "{{ url_for(endpoint_dashboard) }}" +'?subtype='+ d.name +'&from='+d.label+'&to='+d.label; });
{% endif %} {% endif %}
data.forEach(function(d) { data.forEach(function(d) {
@ -384,11 +393,11 @@ function barchart_type_stack(url, id) {
.attr('x', x(d.date) + x.bandwidth()/2) .attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total)) .attr('y', y(d.total))
{% if date_from|string == date_to|string and type is none %} {% if date_from|string == date_to|string and type is none %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date }) .on("click", function () {window.location.href = "{{ url_for(endpoint_dashboard) }}"+'?from={{from}}&to={{date_to}}&subtype='+d.date })
{% elif date_from|string == date_to|string and type is not none %} {% elif date_from|string == date_to|string and type is not none %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date }) .on("click", function () {window.location.href = "{{ url_for(endpoint_dashboard) }}?subtype={{subtype}}&from="+d.date+'&to='+d.date })
{% else %} {% else %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from='+d.date+'&date_to='+d.date }) .on("click", function () {window.location.href = "{{ url_for(endpoint_dashboard) }}"+'?from='+d.date+'&to='+d.date })
{% endif %} {% endif %}
.style("text-anchor", "middle") .style("text-anchor", "middle")
.text(d.total); .text(d.total);

6
var/www/templates/sidebars/sidebar_objects.html
View file

@ -41,19 +41,19 @@
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump"> <a class="nav-link" href="{{url_for('objects_subtypes.objects_dashboard_pgp')}}" id="nav_dashboard_pgpdump">
<i class="fas fa-key"></i> <i class="fas fa-key"></i>
<span>PGP Dumps</span> <span>PGP Dumps</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency"> <a class="nav-link" href="{{url_for('objects_subtypes.objects_dashboard_cryptocurrency')}}" id="nav_dashboard_cryptocurrency">
<i class="fas fa-coins"></i> <i class="fas fa-coins"></i>
<span>Cryptocurrency</span> <span>Cryptocurrency</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.username_page')}}" id="nav_dashboard_username"> <a class="nav-link" href="{{url_for('objects_subtypes.objects_dashboard_username')}}" id="nav_dashboard_username">
<i class="fas fa-user"></i> <i class="fas fa-user"></i>
<span>Username</span> <span>Username</span>
</a> </a>