AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [UI] show correlation (pgp. decoded, cryptocurrency) migration

Browse source
This commit is contained in:
Terrtia 2019-11-15 17:22:50 +01:00
parent bb03ef532b
commit 2e921a12f2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
17 changed files with 948 additions and 1306 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
bin/lib/Correlate_object.py
View file

@ -38,13 +38,13 @@ def get_object_metadata(object_type, correlation_id, type_id=None):
if object_type == 'domain': if object_type == 'domain':
return Domain.Domain(correlation_id).get_domain_metadata() return Domain.Domain(correlation_id).get_domain_metadata()
elif object_type == 'paste': elif object_type == 'paste':
return None return {}
elif object_type == 'decoded': elif object_type == 'decoded':
return Decoded.get_decoded_metadata(correlation_id) return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True)
elif object_type == 'pgp': elif object_type == 'pgp':
return Pgp.pgp._get_metadata(type_id, correlation_id) return Pgp.pgp.get_metadata(type_id, correlation_id)
elif object_type == 'cryptocurrency': elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency._get_metadata(type_id, correlation_id) return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
def get_object_correlation(object_type, value, correlation_names, correlation_objects, requested_correl_type=None): def get_object_correlation(object_type, value, correlation_names, correlation_objects, requested_correl_type=None):
if object_type == 'domain': if object_type == 'domain':
@ -140,14 +140,14 @@ def get_item_url(correlation_name, value, correlation_type=None):
''' '''
url = '#' url = '#'
if correlation_name == "pgp": if correlation_name == "pgp":
endpoint = 'hashDecoded.show_pgpdump' endpoint = 'correlation.show_correlation'
url = url_for(endpoint, type_id=correlation_type, key_id=value) url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'cryptocurrency': elif correlation_name == 'cryptocurrency':
endpoint = 'hashDecoded.show_cryptocurrency' endpoint = 'correlation.show_correlation'
url = url_for(endpoint, type_id=correlation_type, key_id=value) url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'decoded': elif correlation_name == 'decoded':
endpoint = 'hashDecoded.showHash' endpoint = 'correlation.show_correlation'
url = url_for(endpoint, hash=value) url = url_for(endpoint, object_type="decoded", correlation_id=value)
elif correlation_name == 'domain': elif correlation_name == 'domain':
endpoint = 'crawler_splash.showDomain' endpoint = 'crawler_splash.showDomain'
url = url_for(endpoint, domain=value) url = url_for(endpoint, domain=value)

45
bin/lib/Decoded.py
View file

@ -8,6 +8,7 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Item import Item
import Date
import ConfigLoader import ConfigLoader
@ -24,12 +25,54 @@ def get_decoded_item_type(sha1_string):
''' '''
return r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'estimated_type') return r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'estimated_type')
def get_decoded_metadata(sha1_string): def nb_decoded_seen_in_item(sha1_string):
nb = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'nb_seen_in_all_pastes')
if nb is None:
return 0
else:
return int(nb)
def nb_decoded_item_size(sha1_string):
nb = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'size')
if nb is None:
return 0
else:
return int(nb)
def get_decoded_metadata(sha1_string, nb_seen=False, size=False):
metadata_dict = {} metadata_dict = {}
metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen') metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen')
metadata_dict['last_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'last_seen') metadata_dict['last_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'last_seen')
if nb_seen:
metadata_dict['nb_seen'] = nb_decoded_seen_in_item(sha1_string)
if size:
metadata_dict['size'] = nb_decoded_item_size(sha1_string)
return metadata_dict return metadata_dict
def get_list_nb_previous_hash(sha1_string, num_day):
nb_previous_hash = []
for date_day in Date.get_previous_date_list(num_day):
nb_previous_hash.append(get_nb_hash_seen_by_date(sha1_string, date_day))
return nb_previous_hash
def get_nb_hash_seen_by_date(sha1_string, date_day):
nb = r_serv_metadata.zscore('hash_date:{}'.format(date_day), sha1_string)
if nb is None:
return 0
else:
return int(nb)
def get_decoded_vt_report(sha1_string):
vt_dict = {}
res = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'vt_link')
if res:
vt_dict["link"] = res
res = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'vt_report')
if res:
vt_dict["report"] = res
return vt_dict
def get_decoded_items_list(sha1_string): def get_decoded_items_list(sha1_string):
return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1) return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1)

4
bin/lib/Domain.py
View file

@ -14,7 +14,6 @@ import redis
import random import random
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Correlation
import Cryptocurrency import Cryptocurrency
from Pgp import pgp from Pgp import pgp
import Decoded import Decoded
@ -25,6 +24,7 @@ cryptocurrency = Cryptocurrency.cryptocurrency
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
import Correlate_object
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
@ -220,7 +220,7 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
:rtype: dict :rtype: dict
''' '''
if not correlation_names: if not correlation_names:
correlation_names = Correlation.get_all_correlation_names() correlation_names = Correlate_object.get_all_correlation_names()
domain_correl = {} domain_correl = {}
for correlation_name in correlation_names: for correlation_name in correlation_names:
if correlation_name=='cryptocurrency': if correlation_name=='cryptocurrency':

26
bin/packages/Correlation.py
View file

@ -8,6 +8,9 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None config_loader = None
@ -35,8 +38,31 @@ class Correlation(object):
meta_dict = {} meta_dict = {}
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen') meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen') meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
return meta_dict return meta_dict
def get_metadata(self, correlation_type, field_name, date_format='str_date'):
meta_dict = self._get_metadata(correlation_type, field_name)
if date_format == "str_date":
if meta_dict['first_seen']:
meta_dict['first_seen'] = '{}/{}/{}'.format(meta_dict['first_seen'][0:4], meta_dict['first_seen'][4:6], meta_dict['first_seen'][6:8])
if meta_dict['last_seen']:
meta_dict['last_seen'] = '{}/{}/{}'.format(meta_dict['last_seen'][0:4], meta_dict['last_seen'][4:6], meta_dict['last_seen'][6:8])
return meta_dict
def get_nb_object_seen_by_date(self, correlation_type, field_name, date_day):
nb = r_serv_metadata.hget('{}:{}:{}'.format(self.correlation_name, correlation_type, date_day), field_name)
if nb is None:
return 0
else:
return int(nb)
def get_list_nb_previous_correlation_object(self, correlation_type, field_name, numDay):
nb_previous_correlation = []
for date_day in Date.get_previous_date_list(numDay):
nb_previous_correlation.append(self.get_nb_object_seen_by_date(correlation_type, field_name, date_day))
return nb_previous_correlation
def _get_correlation_by_date(self, correlation_type, date): def _get_correlation_by_date(self, correlation_type, date):
return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date)) return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))

11
bin/packages/Date.py
View file

@ -2,6 +2,8 @@
import datetime import datetime
# # TODO: refractor me
class Date(object): class Date(object):
"""docstring for Date""" """docstring for Date"""
def __init__(self, *args): def __init__(self, *args):
@ -52,6 +54,7 @@ def date_substract_day(date, num_day=1):
new_date = str(new_date).replace('-', '') new_date = str(new_date).replace('-', '')
return new_date return new_date
# # TODO: remove me ## FIXME:
def get_date_range(num_day): def get_date_range(num_day):
curr_date = datetime.date.today() curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
@ -61,6 +64,14 @@ def get_date_range(num_day):
date_list.append(date.substract_day(i)) date_list.append(date.substract_day(i))
return list(reversed(date_list)) return list(reversed(date_list))
def get_previous_date_list(num_day):
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = []
for i in range(0, num_day+1):
date_list.append(date.substract_day(i))
return list(reversed(date_list))
def substract_date(date_from, date_to): def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))

55
var/www/blueprints/correlation.py
View file

@ -23,32 +23,25 @@ from Role_Manager import login_admin, login_analyst
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Correlate_object import Correlate_object
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Cryptocurrency
import Pgp
import Decoded
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
vt_enabled = Flask_config.vt_enabled
# ============ BLUEPRINT ============ # ============ BLUEPRINT ============
correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation')) correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
# ============ VARIABLES ============ # ============ VARIABLES ============
def show_correlation(correlation_type, type_id, key_id): ######
if is_valid_type_id(correlation_type, type_id): ### graph_line_json
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id) ### 'hashDecoded.pgpdump_graph_line_json'
if key_id_metadata: ### 'hashDecoded.cryptocurrency_graph_line_json'
###
num_day_sparkline = 6 ######
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id)
return render_template('show_correlation.html', key_id=key_id, type_id=type_id,
correlation_type=correlation_type,
graph_line_endpoint=get_graph_line_json_endpoint(correlation_type),
key_id_metadata=key_id_metadata,
type_icon=get_icon(correlation_type, type_id),
sparkline_values=sparkline_values)
else:
return '404'
else:
return 'error'
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
@ -101,11 +94,30 @@ def sanitise_correlation_objects(correlation_objects):
else: else:
return all_correlation_objects return all_correlation_objects
def get_card_metadata(object_type, correlation_id, type_id=None):
card_dict = {}
if object_type == 'cryptocurrency':
card_dict["sparkline"] = Cryptocurrency.cryptocurrency.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
elif object_type == 'pgp':
card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
elif object_type == 'decoded':
card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id)
card_dict["vt"]["status"] = vt_enabled
elif object_type == 'domain':
pass
elif object_type == 'paste':
pass
return card_dict
# ============= ROUTES ============== # ============= ROUTES ==============
@correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST @correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST
@login_required @login_required
@login_analyst @login_analyst
def showCorrelationDomain(): def show_correlation():
if request.method == 'POST': if request.method == 'POST':
object_type = request.form.get('object_type') object_type = request.form.get('object_type')
type_id = request.form.get('type_id') type_id = request.form.get('type_id')
@ -143,7 +155,7 @@ def showCorrelationDomain():
correlation_objects = ",".join(correlation_objects) correlation_objects = ",".join(correlation_objects)
# redirect to keep history and bookmark # redirect to keep history and bookmark
return redirect(url_for('correlation.showCorrelationDomain', object_type=object_type, type_id=type_id, correlation_id=correlation_id, mode=mode, return redirect(url_for('correlation.show_correlation', object_type=object_type, type_id=type_id, correlation_id=correlation_id, mode=mode,
max_nodes=max_nodes, correlation_names=correlation_names, correlation_objects=correlation_objects)) max_nodes=max_nodes, correlation_names=correlation_names, correlation_objects=correlation_objects))
# request.method == 'GET' # request.method == 'GET'
@ -167,6 +179,7 @@ def showCorrelationDomain():
dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id)
if type_id: if type_id:
dict_object["metadata"]['type_id'] = type_id dict_object["metadata"]['type_id'] = type_id
dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id)
return render_template("show_correlation.html", dict_object=dict_object) return render_template("show_correlation.html", dict_object=dict_object)
@correlation.route('/correlation/graph_node_json') @correlation.route('/correlation/graph_node_json')

197
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -224,13 +224,7 @@ def get_correlation_type_page_endpoint(correlation_type):
return endpoint return endpoint
def get_show_key_id_endpoint(correlation_type): def get_show_key_id_endpoint(correlation_type):
if correlation_type == 'pgpdump': return 'correlation.show_correlation'
endpoint = 'hashDecoded.show_pgpdump'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.show_cryptocurrency'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_range_type_json_endpoint(correlation_type): def get_range_type_json_endpoint(correlation_type):
if correlation_type == 'pgpdump': if correlation_type == 'pgpdump':
@ -351,8 +345,13 @@ def main_correlation_page(correlation_type, type_id, date_from, date_to, show_de
l_type = get_all_types_id(correlation_type) l_type = get_all_types_id(correlation_type)
correlation_type_n = correlation_type
if correlation_type_n=='pgpdump':
correlation_type_n = 'pgp'
return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata, return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata,
correlation_type=correlation_type, correlation_type=correlation_type,
correlation_type_n=correlation_type_n,
correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type), correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type),
correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type), correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type),
show_key_id_endpoint=get_show_key_id_endpoint(correlation_type), show_key_id_endpoint=get_show_key_id_endpoint(correlation_type),
@ -362,27 +361,27 @@ def main_correlation_page(correlation_type, type_id, date_from, date_to, show_de
date_from=date_from, date_to=date_to, date_from=date_from, date_to=date_to,
show_decoded_files=show_decoded_files) show_decoded_files=show_decoded_files)
def show_correlation(correlation_type, type_id, key_id): # def show_correlation(correlation_type, type_id, key_id):
if is_valid_type_id(correlation_type, type_id): # if is_valid_type_id(correlation_type, type_id):
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id) # key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id)
if key_id_metadata: # if key_id_metadata:
#
num_day_sparkline = 6 # num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline) # date_range_sparkline = get_date_range(num_day_sparkline)
#
sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id) # sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id)
return render_template('showCorrelation.html', key_id=key_id, type_id=type_id, # return render_template('showCorrelation.html', key_id=key_id, type_id=type_id,
correlation_type=correlation_type, # correlation_type=correlation_type,
graph_node_endpoint=get_graph_node_json_endpoint(correlation_type), # graph_node_endpoint=get_graph_node_json_endpoint(correlation_type),
graph_line_endpoint=get_graph_line_json_endpoint(correlation_type), # graph_line_endpoint=get_graph_line_json_endpoint(correlation_type),
font_family=get_font_family(correlation_type), # font_family=get_font_family(correlation_type),
key_id_metadata=key_id_metadata, # key_id_metadata=key_id_metadata,
type_icon=get_icon(correlation_type, type_id), # type_icon=get_icon(correlation_type, type_id),
sparkline_values=sparkline_values) # sparkline_values=sparkline_values)
else: # else:
return '404' # return '404'
else: # else:
return 'error' # return 'error'
def correlation_type_range_type_json(correlation_type, date_from, date_to): def correlation_type_range_type_json(correlation_type, date_from, date_to):
date_range = [] date_range = []
@ -620,60 +619,60 @@ def hash_hash():
hash = request.args.get('hash') hash = request.args.get('hash')
return render_template('hash_hash.html') return render_template('hash_hash.html')
#
@hashDecoded.route('/hashDecoded/showHash') # @hashDecoded.route('/hashDecoded/showHash')
@login_required # @login_required
@login_analyst # @login_analyst
def showHash(): # def showHash():
hash = request.args.get('hash') # hash = request.args.get('hash')
#hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea' # #hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea'
#
# TODO FIXME show error # # TODO FIXME show error
if hash is None: # if hash is None:
return hashDecoded_page() # return hashDecoded_page()
#
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') # estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
# hash not found # # hash not found
# TODO FIXME show error # # TODO FIXME show error
if estimated_type is None: # if estimated_type is None:
return hashDecoded_page() # return hashDecoded_page()
#
else: # else:
file_icon = get_file_icon(estimated_type) # file_icon = get_file_icon(estimated_type)
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size') # size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen') # first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen') # last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') # nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
#
# get all encoding for this hash # # get all encoding for this hash
list_hash_decoder = [] # list_hash_decoder = []
list_decoder = r_serv_metadata.smembers('all_decoder') # list_decoder = r_serv_metadata.smembers('all_decoder')
for decoder in list_decoder: # for decoder in list_decoder:
encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder') # encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder')
if encoding is not None: # if encoding is not None:
list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding}) # list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding})
#
num_day_type = 6 # num_day_type = 6
date_range_sparkline = get_date_range(num_day_type) # date_range_sparkline = get_date_range(num_day_type)
sparkline_values = list_sparkline_values(date_range_sparkline, hash) # sparkline_values = list_sparkline_values(date_range_sparkline, hash)
#
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): # if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
b64_vt = True # b64_vt = True
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') # b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') # b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
else: # else:
b64_vt = False # b64_vt = False
b64_vt_link = '' # b64_vt_link = ''
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') # b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
# hash never refreshed # # hash never refreshed
if b64_vt_report is None: # if b64_vt_report is None:
b64_vt_report = '' # b64_vt_report = ''
#
return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link, # return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link,
b64_vt_report=b64_vt_report, # b64_vt_report=b64_vt_report,
size=size, estimated_type=estimated_type, file_icon=file_icon, # size=size, estimated_type=estimated_type, file_icon=file_icon,
first_seen=first_seen, list_hash_decoder=list_hash_decoder, # first_seen=first_seen, list_hash_decoder=list_hash_decoder,
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values) # last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
@hashDecoded.route('/hashDecoded/downloadHash') @hashDecoded.route('/hashDecoded/downloadHash')
@ -1207,22 +1206,22 @@ def all_cryptocurrency_search():
show_decoded_files = request.form.get('show_decoded_files') show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route('/correlation/show_pgpdump') # @hashDecoded.route('/correlation/show_pgpdump')
@login_required # @login_required
@login_analyst # @login_analyst
def show_pgpdump(): # def show_pgpdump():
type_id = request.args.get('type_id') # type_id = request.args.get('type_id')
key_id = request.args.get('key_id') # key_id = request.args.get('key_id')
return show_correlation('pgpdump', type_id, key_id) # return show_correlation('pgpdump', type_id, key_id)
#
#
@hashDecoded.route('/correlation/show_cryptocurrency') # @hashDecoded.route('/correlation/show_cryptocurrency')
@login_required # @login_required
@login_analyst # @login_analyst
def show_cryptocurrency(): # def show_cryptocurrency():
type_id = request.args.get('type_id') # type_id = request.args.get('type_id')
key_id = request.args.get('key_id') # key_id = request.args.get('key_id')
return show_correlation('cryptocurrency', type_id, key_id) # return show_correlation('cryptocurrency', type_id, key_id)
@hashDecoded.route('/correlation/cryptocurrency_range_type_json') @hashDecoded.route('/correlation/cryptocurrency_range_type_json')
@login_required @login_required
@ -1248,6 +1247,7 @@ def pgpdump_graph_node_json():
key_id = request.args.get('key_id') key_id = request.args.get('key_id')
return correlation_graph_node_json('pgpdump', type_id, key_id) return correlation_graph_node_json('pgpdump', type_id, key_id)
# # TODO: REFRACTOR
@hashDecoded.route('/correlation/cryptocurrency_graph_node_json') @hashDecoded.route('/correlation/cryptocurrency_graph_node_json')
@login_required @login_required
@login_analyst @login_analyst
@ -1256,6 +1256,7 @@ def cryptocurrency_graph_node_json():
key_id = request.args.get('key_id') key_id = request.args.get('key_id')
return correlation_graph_node_json('cryptocurrency', type_id, key_id) return correlation_graph_node_json('cryptocurrency', type_id, key_id)
# # TODO: REFRACTOR
@hashDecoded.route('/correlation/pgpdump_graph_line_json') @hashDecoded.route('/correlation/pgpdump_graph_line_json')
@login_required @login_required
@login_analyst @login_analyst

5
var/www/modules/hashDecoded/templates/DaysCorrelation.html
View file

@ -145,7 +145,7 @@
{% for key_id in all_metadata %} {% for key_id in all_metadata %}
<tr> <tr>
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td> <td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td>
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type_id={{ all_metadata[key_id]['type_id'] }}&key_id={{ key_id }}">{{ key_id }}</a></td> <td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?object_type={{correlation_type_n}}&type_id={{ all_metadata[key_id]['type_id'] }}&correlation_id={{ key_id }}&correlation_objects=paste">{{ key_id }}</a></td>
<td>{{ all_metadata[key_id]['first_seen'] }}</td> <td>{{ all_metadata[key_id]['first_seen'] }}</td>
<td>{{ all_metadata[key_id]['last_seen'] }}</td> <td>{{ all_metadata[key_id]['last_seen'] }}</td>
<td>{{ all_metadata[key_id]['nb_seen'] }}</td> <td>{{ all_metadata[key_id]['nb_seen'] }}</td>
@ -224,9 +224,6 @@
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id'); chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
{% endif %} {% endif %}
//draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
//draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
chart.onResize(); chart.onResize();
$(window).on("resize", function() { $(window).on("resize", function() {
chart.onResize(); chart.onResize();

2
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -158,7 +158,7 @@
{% for b64 in l_64 %} {% for b64 in l_64 %}
<tr> <tr>
<td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td> <td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ b64[2] }}">{{ b64[2] }}</a></td> <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a></td>
<td>{{ b64[5] }}</td> <td>{{ b64[5] }}</td>
<td>{{ b64[6] }}</td> <td>{{ b64[6] }}</td>
<td>{{ b64[3] }}</td> <td>{{ b64[3] }}</td>

512
var/www/modules/hashDecoded/templates/showCorrelation.html
View file

@ -1,512 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>AIL - framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
line.link {
stroke: #666;
}
line.link:hover{
stroke: red;
stroke-width: 2px
}
.node {
pointer-events: all;
}
circle {
stroke: none;
}
.graph_text_node {
font: 8px sans-serif;
pointer-events: none;
}
.graph_node_icon {
pointer-events: none;
}
.node text {
font: 8px sans-serif;
pointer-events: auto;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
.graph_panel {
padding: unset;
}
.line_graph {
fill: none;
stroke: steelblue;
stroke-width: 2px;
stroke-linejoin: round;
stroke-linecap: round;
stroke-width: 1.5;
/*attr('stroke', '#bcbd22').*/
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'decoded/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ key_id }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>type</th>
<th>First_seen</th>
<th>Last_seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td><i class="{{ type_icon }}"></i>&nbsp;&nbsp;{{ type_id }}</td>
<td>{{ key_id_metadata['first_seen'] }}</td>
<td>{{ key_id_metadata['last_seen'] }}</td>
<td>{{ key_id_metadata['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
</div>
</div>
<div class="row">
<div class="col-xl-10">
<div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body graph_panel">
<div id="graph">
</div>
</div>
</div>
</div>
<div class="col-xl-2">
<div class="card my-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body text-center px-0 py-0">
<button class="btn btn-primary my-4" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
{% if correlation_type=='pgpdump' %}
{% include 'decoded/show_helper_pgpdump.html' %}
{% elif correlation_type=='cryptocurrency' %}
{% include 'decoded/show_helper_cryptocurrency.html' %}
{% endif %}
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-header">
<i class="fas fa-chart-bar"></i> Graph
</div>
<div class="panel-body ">
<div id="graph_line">
</div>
</div>
</div>
</div>
</div>
</div>
<script>
var all_graph = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
sparkline("sparkline", {{ sparkline_values }}, {});
all_graph.node_graph = create_graph("{{ url_for(graph_node_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for(graph_line_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
all_graph.onResize();
});
$(window).on("resize", function() {
all_graph.onResize();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function resize_graph() {
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
}
</script>
<script>
var width = 400,
height = 400;
var link;
var zoom = d3.zoom()
.scaleExtent([.2, 10])
.on("zoom", zoomed);
//var transform = d3.zoomIdentity;
var color = d3.scaleOrdinal(d3.schemeCategory10);
var div = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) { return d.id; }))
.force("charge", d3.forceManyBody())
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
.attr("height", height)
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
.on("dblclick.zoom", null)
var container_graph = svg_node.append("g");
//.attr("transform", "translate(40,0)")
//.attr("transform", "scale(2)");
function create_graph(url){
d3.json(url)
.then(function(data){
link = container_graph.append("g")
.selectAll("line")
.data(data.links)
.enter().append("line")
.attr("class", "link");
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
var node = container_graph.selectAll(".node")
.data(data.nodes)
.enter().append("g")
.attr("class", "nodes")
.on("dblclick", doubleclick)
.on("click", click)
.on("mouseover", mouseovered)
.on("mouseout", mouseouted)
.call(d3.drag()
.on("start", drag_start)
.on("drag", dragged)
.on("end", drag_end));
node.append("circle")
.attr("r", function(d) {
return (d.hash) ? 6 : 5; })
.attr("fill", function(d) {
if(!d.hash){ return color(d.group);}
if(d.group == 1){ return "orange";}
return "rgb(141, 211, 199)"; });
node.append('text')
.attr('text-anchor', 'middle')
.attr('dominant-baseline', 'central')
.attr("class", "graph_node_icon {{font_family}}")
.attr('font-size', '8px' )
.attr('pointer-events', 'none')
.text(function(d) {
if(d.hash){
return d.icon
} });
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
/* node.append("text")
.attr("dy", 3)
.attr("dx", 7)
.attr("class", "graph_text_node")
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.id; });*/
simulation
.nodes(data.nodes)
.on("tick", ticked);
simulation.force("link")
.links(data.links);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
/*node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });*/
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
}
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
}
function doubleclick (d) {
window.open(d.url, '_blank');
}
function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
// tooltip
var content;
if(d.hash == true){
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>First seen</i>: <span id='tooltip-id-first_seen'></span><br/>"+
"<i>Last seen</i>: <span id='tooltip-id-last_seen'></span><br/>"+
"<i>nb_seen</i>: <span id='tooltip-id-nb_seen'></span>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
$("#tooltip-id-first_seen").text(d.first_seen);
$("#tooltip-id-last_seen").text(d.last_seen);
$("#tooltip-id-nb_seen").text(d.nb_seen_in_paste);
} else {
content = "<b><span id='tooltip-id-name'></span></b><br/>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
}
//links
/*link.style("stroke-opacity", function(o) {
return o.source === d || o.target === d ? 1 : opacity;
});*/
link.style("stroke", function(o){
return o.source === d || o.target === d ? "#666" : "#ddd";
});
}
function mouseouted() {
div.transition()
.duration(500)
.style("opacity", 0);
link.style("stroke", "#666");
}
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>
</body>
</html>

587
var/www/modules/hashDecoded/templates/showHash.html
View file

@ -1,587 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Hash Information - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
line.link {
stroke: #666;
}
line.link:hover{
stroke: red;
stroke-width: 2px
}
.node {
pointer-events: all;
}
circle {
stroke: none;
}
.graph_text_node {
font: 8px sans-serif;
pointer-events: none;
}
.graph_node_icon {
pointer-events: none;
}
.node text {
font: 8px sans-serif;
pointer-events: auto;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
.graph_panel {
padding: unset;
}
.line_graph {
fill: none;
stroke: steelblue;
stroke-width: 2px;
stroke-linejoin: round;
stroke-linecap: round;
stroke-width: 1.5;
/*attr('stroke', '#bcbd22').*/
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'decoded/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ hash }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Estimated type</th>
<th>First_seen</th>
<th>Last_seen</th>
<th>Size (Kb)</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td><i class="fas {{ file_icon }}"></i>&nbsp;&nbsp;{{ estimated_type }}</td>
<td>{{ first_seen }}</td>
<td>{{ last_seen }}</td>
<td>{{ size }}</td>
<td>{{ nb_seen_in_all_pastes }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
{% if vt_enabled %}
{% if not b64_vt %}
<darkbutton>
<button id="submit_vt_b" class="btn btn-primary" onclick="sendFileToVT('{{ hash }}')" style="font-size: 15px">
<i class="fas fa-paper-plane"></i>&nbsp;Send this file to VT
</button>
</darkbutton>
{% else %}
<a class="btn btn-primary" target="_blank" href="{{ b64_vt_link }}" style="font-size: 15px"><i class="fas fa-link"></i> VT Report</a>
{% endif %}
<button class="btn btn-outline-secondary" onclick="updateVTReport('{{ hash }}')" style="font-size: 15px">
<div id="report_vt_b"><i class="fas fa-sync-alt"></i>&nbsp;{{ b64_vt_report }}</div>
</button>
{% else %}
Virus Total submission is disabled
{% endif %}
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{hash}}" target="blank" class="float-right" style="font-size: 15px">
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
</button>
</a>
</div>
</div>
<div class="row">
<div class="col-xl-10">
<div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body graph_panel">
<div id="graph">
</div>
</div>
</div>
</div>
<div class="col-xl-2">
<div class="card">
<div class="card-header">
<i class="fas fa-unlock-alt" aria-hidden="true"></i> Encoding
</div>
<div class="card-body text-center">
{% for encoding in list_hash_decoder %}
<button class="btn btn-outline-dark" disabled>
{{encoding['encoding']}} <span class="badge badge-dark">{{encoding['nb_seen']}}</span>
</button>
{% endfor %}
</div>
</div>
<div class="card my-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body text-center px-0 py-0">
<button class="btn btn-primary my-4" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
<ul class="list-group">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
<li class="list-group-item text-left">
<p>Double click on a node to open Hash/Paste<br><br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
Current Hash<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
Hashes<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
Pastes
</p>
</li>
<li class="list-group-item list-group-item-info">
Hash Types:
</li>
<li class="list-group-item text-left">
<i class="fas fa-file"></i> Application<br>
<i class="fas fa-file-video"></i> Audio<br>
<i class="fas fa-file-image"></i> Image<br>
<i class="fas fa-file-alt"></i> Text<br>
<i class="fas fa-sticky-note"></i> Other
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-header">
<i class="fas fa-chart-bar"></i> Graph
</div>
<div class="panel-body ">
<div id="graph_line">
</div>
</div>
</div>
</div>
</div>
</div>
<script>
var all_graph = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
sparkline("sparkline", {{ sparkline_values }}, {});
all_graph.node_graph = create_graph("{{ url_for('hashDecoded.hash_graph_node_json') }}?hash={{hash}}");
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{hash}}");
all_graph.onResize();
});
$(window).on("resize", function() {
all_graph.onResize();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function sendFileToVT(hash) {
//send file to vt
$.getJSON("{{ url_for('hashDecoded.send_file_to_vt_js') }}?hash="+hash,
function(data) {
var content = '<a id="submit_vt_b" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fas fa-link"> '+ ' VT Report' +'</i></a>';
$('#submit_vt_b').remove();
$('darkbutton').append(content);
});
}
function updateVTReport(hash) {
//updateReport
$.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
function(data) {
var content = '<i class="fas fa-sync-alt"></i> ' +data['report_vt'];
$( "#report_vt_b" ).html(content);
});
}
</script>
<script>
function resize_graph() {
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
}
</script>
<script>
var width = 400,
height = 400;
var link;
var zoom = d3.zoom()
.scaleExtent([.2, 10])
.on("zoom", zoomed);
//var transform = d3.zoomIdentity;
var color = d3.scaleOrdinal(d3.schemeCategory10);
var div = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) { return d.id; }))
.force("charge", d3.forceManyBody())
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
.attr("height", height)
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
.on("dblclick.zoom", null)
var container_graph = svg_node.append("g");
//.attr("transform", "translate(40,0)")
//.attr("transform", "scale(2)");
function create_graph(url){
d3.json(url)
.then(function(data){
link = container_graph.append("g")
.selectAll("line")
.data(data.links)
.enter().append("line")
.attr("class", "link");
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
var node = container_graph.selectAll(".node")
.data(data.nodes)
.enter().append("g")
.attr("class", "nodes")
.on("dblclick", doubleclick)
.on("click", click)
.on("mouseover", mouseovered)
.on("mouseout", mouseouted)
.call(d3.drag()
.on("start", drag_start)
.on("drag", dragged)
.on("end", drag_end));
node.append("circle")
.attr("r", function(d) {
return (d.hash) ? 6 : 5; })
.attr("fill", function(d) {
if(!d.hash){ return color(d.group);}
if(d.group == 1){ return "orange";}
return "rgb(141, 211, 199)"; });
node.append('text')
.attr('text-anchor', 'middle')
.attr('dominant-baseline', 'central')
.attr("class", "graph_node_icon fa")
.attr('font-size', '8px' )
.attr('pointer-events', 'none')
.text(function(d) {
if(d.hash){
return d.icon
} });
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
/* node.append("text")
.attr("dy", 3)
.attr("dx", 7)
.attr("class", "graph_text_node")
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.id; });*/
simulation
.nodes(data.nodes)
.on("tick", ticked);
simulation.force("link")
.links(data.links);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
/*node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });*/
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
}
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
}
function doubleclick (d) {
window.open(d.url, '_blank');
}
function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
// tooltip
var content;
if(d.hash == true){
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>First seen</i>: <span id='tooltip-id-first_seen'></span><br/>"+
"<i>Last seen</i>: <span id='tooltip-id-last_seen'></span><br/>"+
"<i>nb_seen</i>: <span id='tooltip-id-nb_seen'></span>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
$("#tooltip-id-first_seen").text(d.first_seen);
$("#tooltip-id-last_seen").text(d.last_seen);
$("#tooltip-id-nb_seen").text(d.nb_seen_in_paste);
} else {
content = "<b><span id='tooltip-id-name'></span></b><br/>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
}
//links
/*link.style("stroke-opacity", function(o) {
return o.source === d || o.target === d ? 1 : opacity;
});*/
link.style("stroke", function(o){
return o.source === d || o.target === d ? "#666" : "#ddd";
});
}
function mouseouted() {
div.transition()
.duration(500)
.style("opacity", 0);
link.style("stroke", "#666");
}
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>
</body>
</html>

168
var/www/templates/correlation/legend_graph_correlation.html Normal file
View file

@ -0,0 +1,168 @@
<table class="table table-bordered">
<thead>
<tr class="table-info"">
<th>
Cryptocurrency:
</th>
<th class="">
Decoded:
</th>
<th class="">
Pgp:
</th>
<th class="">
Domain:
</th>
<th class="">
Paste:
</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf15a;</text>
</g>
</svg>
bitcoin
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf3d0;</text>
</g>
</svg>
monero
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf42e;</text>
</g>
</svg>
etherum
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf51e;</text>
</g>
</svg>
other cryptocurrencies
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15b;</text>
</g>
</svg>
application
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1c7;</text>
</g>
</svg>
audio
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15c;</text>
</g>
</svg>
text
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf249;</text>
</g>
</svg>
other types of file
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf084;</text>
</g>
</svg>
key
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
</g>
</svg>
name
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1fa;</text>
</g>
</svg>
mail
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf06e;</text>
</g>
</svg>
onion
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf13b;</text>
</g>
</svg>
web
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="red"></circle>
</g>
</svg>
crawled
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#332288"></circle>
</g>
</svg>
other
</div>
</td>
</tr>
</tbody>
</table>

186
var/www/templates/correlation/metadata_card_decoded.html Normal file
View file

@ -0,0 +1,186 @@
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>Estimated type</th>
<th>First seen</th>
<th>Last seen</th>
<th>Size (Kb)</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ dict_object["object_type"] }}</td>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
</g>
</svg>
{{ dict_object["metadata"]["type_id"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
<td>{{ dict_object["metadata"]['size'] }}</td>
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
{% if dict_object["metadata_card"]["vt"]["status"] %}
{% if not "link" in dict_object["metadata_card"]["vt"] %}
<darkbutton>
<button id="submit_vt_b" class="btn btn-primary" onclick="sendFileToVT('{{ dict_object["correlation_id"] }}')" style="font-size: 15px">
<i class="fas fa-paper-plane"></i>&nbsp;Send this file to VT
</button>
</darkbutton>
{% else %}
<a class="btn btn-primary" target="_blank" href="{{ dict_object["metadata_card"]["vt"]["link"] }}" style="font-size: 15px"><i class="fas fa-link"></i> VT Report</a>
{% endif %}
<button class="btn btn-outline-secondary" onclick="updateVTReport('{{ dict_object["correlation_id"] }}')" style="font-size: 15px">
<div id="report_vt_b"><i class="fas fa-sync-alt"></i>&nbsp;{{ dict_object["metadata_card"]["vt"]["report"] }}</div>
</button>
{% else %}
Virus Total submission is disabled
{% endif %}
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{ dict_object["correlation_id"] }}" target="blank" class="float-right" style="font-size: 15px">
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
</button>
</a>
</div>
</div>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<script>
sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
</script>
<script>
function sendFileToVT(hash) {
//send file to vt
$.getJSON("{{ url_for('hashDecoded.send_file_to_vt_js') }}?hash="+hash,
function(data) {
var content = '<a id="submit_vt_b" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fas fa-link"> '+ ' VT Report' +'</i></a>';
$('#submit_vt_b').remove();
$('darkbutton').append(content);
});
}
function updateVTReport(hash) {
//updateReport
$.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
function(data) {
var content = '<i class="fas fa-sync-alt"></i> ' +data['report_vt'];
$( "#report_vt_b" ).html(content);
});
}
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>

140
var/www/templates/correlation/metadata_card_pgp.html Normal file
View file

@ -0,0 +1,140 @@
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>type</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ dict_object["object_type"] }}</td>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
</g>
</svg>
{{ dict_object["metadata"]["type_id"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
</div>
</div>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<script>
sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>

101
var/www/templates/correlation/show_correlation.html
View file

@ -18,6 +18,11 @@
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style> <style>
.icon_legend {
color: #0c5460;
background-color: #bee5eb;
}
line.link { line.link {
stroke: #666; stroke: #666;
} }
@ -85,44 +90,11 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="card my-3"> {% if dict_object["object_type"] in ["pgp", "cryptocurrency"] %}
<div class="card-header" style="background-color:#d9edf7;font-size: 15px"> {% include 'correlation/metadata_card_pgp.html' %}
<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4> {% elif dict_object["object_type"] == "decoded" %}
<ul class="list-group mb-2"> {% include 'correlation/metadata_card_decoded.html' %}
<li class="list-group-item py-0"> {% endif %}
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>type</th>
{% if dict_object["metadata"]['first_seen'] %}
<th>First_seen</th>
<th>Last_seen</th>
{% endif %}
</tr>
</thead>
<tbody>
<tr>
<td>{{ dict_object["object_type"] }}</td>
<td><i class="fas fa-key"></i>&nbsp;&nbsp;{{ dict_object["metadata"]["type_id"] }}</td>
{% if dict_object["metadata"]['first_seen'] %}
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
{% endif %}
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
</div>
</div>
<div class="row"> <div class="row">
<div class="col-xl-10"> <div class="col-xl-10">
@ -130,6 +102,11 @@
<div class="card mb-3"> <div class="card mb-3">
<div class="card-header"> <div class="card-header">
<i class="fas fa-project-diagram"></i> Graph <i class="fas fa-project-diagram"></i> Graph
<span class="float-right">
<button class="btn btn-primary py-1" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
</span>
</div> </div>
<div class="card-body graph_panel"> <div class="card-body graph_panel">
<div id="graph"> <div id="graph">
@ -141,19 +118,13 @@
<div class="col-xl-2"> <div class="col-xl-2">
<div class="card mb-3"> <div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body text-center px-0 py-0"> <div class="card-body text-center px-0 py-0">
<button class="btn btn-primary my-4" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
<ul class="list-group"> <ul class="list-group">
<li class="list-group-item list-group-item-info">Select Correlation</i></li> <li class="list-group-item list-group-item-info">Select Correlation</i></li>
<li class="list-group-item text-left"> <li class="list-group-item text-left">
<form action="{{ url_for('correlation.showCorrelationDomain') }}" method="post"> <form action="{{ url_for('correlation.show_correlation') }}" method="post">
<input type="hidden" id="object_type" name="object_type" value="{{ dict_object["object_type"] }}"> <input type="hidden" id="object_type" name="object_type" value="{{ dict_object["object_type"] }}">
<input type="hidden" id="type_id" name="type_id" value="{{ dict_object["metadata"]["type_id"] }}"> <input type="hidden" id="type_id" name="type_id" value="{{ dict_object["metadata"]["type_id"] }}">
<input type="hidden" id="correlation_id" name="correlation_id" value="{{ dict_object["correlation_id"] }}"> <input type="hidden" id="correlation_id" name="correlation_id" value="{{ dict_object["correlation_id"] }}">
@ -205,13 +176,44 @@
</li> </li>
</ul> </ul>
{% include 'decoded/show_helper_pgpdump.html' %} <ul class="list-group">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
<li class="list-group-item text-left">
<p>Double click on a node to open this object<br><br>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
</g>
</svg>
Current Correlation<br>
</p>
</li>
</ul>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="row">
<div class="col-xl-10">
{% include 'correlation/legend_graph_correlation.html' %}
</div>
</div>
{% if dict_object["object_type"] in ["decoded", "pgp", "cryptocurrency"] %}
<div class="card">
<div class="card-header">
<i class="fas fa-chart-bar"></i> Graph
</div>
<div class="panel-body ">
<div id="graph_line">
</div>
</div>
</div>
{% endif %}
</div> </div>
</div> </div>
@ -223,6 +225,13 @@ $(document).ready(function(){
$("#page-Decoded").addClass("active"); $("#page-Decoded").addClass("active");
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?correlation_id={{ dict_object["correlation_id"] }}&object_type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&correlation_names={{ dict_object["correlation_names_str"] }}&correlation_objects={{ dict_object["correlation_objects_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&type_id={{ dict_object["metadata"]["type_id"] }}{% endif %}"); all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?correlation_id={{ dict_object["correlation_id"] }}&object_type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&correlation_names={{ dict_object["correlation_names_str"] }}&correlation_objects={{ dict_object["correlation_objects_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&type_id={{ dict_object["metadata"]["type_id"] }}{% endif %}");
{% if dict_object["object_type"] == "pgp" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.pgpdump_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "cryptocurrency" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.cryptocurrency_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "decoded" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{dict_object["correlation_id"]}}");
{% endif %}
all_graph.onResize(); all_graph.onResize();
}); });

6
var/www/templates/crawler/crawler_splash/showDomain.html
View file

@ -131,7 +131,7 @@
<tbody> <tbody>
{% for decoded in dict_domain['decoded']%} {% for decoded in dict_domain['decoded']%}
<tr> <tr>
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ decoded }}">{{ decoded }}</a></td> <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ decoded }}&correlation_objects=domain">{{ decoded }}</a></td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
@ -186,7 +186,7 @@
<i class="{{ var_icon }}"></i> <i class="{{ var_icon }}"></i>
&nbsp;&nbsp;{{ dict_key }} &nbsp;&nbsp;{{ dict_key }}
</td> </td>
<td><a target="_blank" href="{{ url_for('hashDecoded.show_pgpdump') }}?type_id={{ dict_key }}&key_id={{ key_id }}">{{ key_id }}</a></td> <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=pgp&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
</tr> </tr>
{% endfor %} {% endfor %}
{% endif %} {% endif %}
@ -243,7 +243,7 @@
<i class="{{ var_icon }}"></i> <i class="{{ var_icon }}"></i>
&nbsp;&nbsp;{{ dict_key }} &nbsp;&nbsp;{{ dict_key }}
</td> </td>
<td><a target="_blank" href="{{ url_for('hashDecoded.show_cryptocurrency') }}?type_id={{ dict_key }}&key_id={{ key_id }}">{{ key_id }}</a></td> <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=pgp&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
</tr> </tr>
{% endfor %} {% endfor %}
{% endif %} {% endif %}

189
var/www/templates/decoded/show_helper_pgpdump.html
View file

@ -1,21 +1,168 @@
<ul class="list-group"> <table class="table table-bordered">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li> <thead>
<li class="list-group-item text-left"> <tr class="table-info"">
<p>Double click on a node to open PgpDump/Paste<br><br> <th>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg> Cryptocurrency:
Current PgpDump<br> </th>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg> <th class="">
PgpDump<br> Decoded:
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg> </th>
Pastes <th class="">
</p> Pgp:
</li> </th>
<li class="list-group-item list-group-item-info"> <th class="">
PgpDump Types: Domain:
</li> </th>
<li class="list-group-item text-left"> <th class="">
<i class="fas fa-key"></i> Key<br> Paste:
<i class="fas fa-user-tag"></i> Name<br> </th>
<i class="fas fa-at"></i> Mail<br> </tr>
</li> </thead>
</ul> <tbody>
<tr>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf15a;</text>
</g>
</svg>
bitcoin
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf3d0;</text>
</g>
</svg>
monero
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf42e;</text>
</g>
</svg>
etherum
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf51e;</text>
</g>
</svg>
other cryptocurrencies
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15b;</text>
</g>
</svg>
application
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1c7;</text>
</g>
</svg>
audio
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15c;</text>
</g>
</svg>
text
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf249;</text>
</g>
</svg>
other types of file
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf084;</text>
</g>
</svg>
key
</div>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
</g>
</svg>
name
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1fa;</text>
</g>
</svg>
mail
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf06e;</text>
</g>
</svg>
onion
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf13b;</text>
</g>
</svg>
web
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="red"></circle>
</g>
</svg>
crawled
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#332288"></circle>
</g>
</svg>
other
</div>
</td>
</tr>
</tbody>
</table>