AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [correlation] add cryptocurrency + refractor correlation

Browse source
This commit is contained in:
Terrtia 2019-05-24 12:02:43 +02:00
parent ff23a2bce2
commit 4794cafe01
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
8 changed files with 1745 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
OVERVIEW.md
View file

@ -157,50 +157,50 @@ Redis and ARDB overview
##### Hset: ##### Hset:
| Key | Field | Value | | Key | Field | Value |
| ------ | ------ | ------ | | ------ | ------ | ------ |
| pgp_metadata_key:*key id* | first_seen | **date** | | pgpdump_metadata_key:*key id* | first_seen | **date** |
| | last_seen | **date** | | | last_seen | **date** |
| | | | | |
| pgp_metadata_name:*name* | first_seen | **date** | | pgpdump_metadata_name:*name* | first_seen | **date** |
| | last_seen | **date** | | | last_seen | **date** |
| | | | | |
| pgp_metadata_mail:*mail* | first_seen | **date** | | pgpdump_metadata_mail:*mail* | first_seen | **date** |
| | last_seen | **date** | | | last_seen | **date** |
##### set: ##### set:
| Key | Value | | Key | Value |
| ------ | ------ | | ------ | ------ |
| pgp_key:*key id* | *item_path* | | set_pgpdump_key:*key id* | *item_path* |
| | | | | |
| pgp_name:*name* | *item_path* | | set_pgpdump_name:*name* | *item_path* |
| | | | | |
| pgp_mail:*mail* | *item_path* | | set_pgpdump_mail:*mail* | *item_path* |
##### Hset date: ##### Hset date:
| Key | Field | Value | | Key | Field | Value |
| ------ | ------ | | ------ | ------ |
| pgp:key:*date* | *key* | *nb seen* | | pgpdump:key:*date* | *key* | *nb seen* |
| | | | | |
| pgp:name:*date* | *name* | *nb seen* | | pgpdump:name:*date* | *name* | *nb seen* |
| | | | | |
| pgp:mail:*date* | *mail* | *nb seen* | | pgpdump:mail:*date* | *mail* | *nb seen* |
##### zset: ##### zset:
| Key | Field | Value | | Key | Field | Value |
| ------ | ------ | ------ | | ------ | ------ | ------ |
| pgp_all:key | *key* | *nb seen* | | pgpdump_all:key | *key* | *nb seen* |
| | | | | |
| pgp_all:name | *name* | *nb seen* | | pgpdump_all:name | *name* | *nb seen* |
| | | | | |
| pgp_all:mail | *mail* | *nb seen* | | pgpdump_all:mail | *mail* | *nb seen* |
##### set: ##### set:
| Key | Value | | Key | Value |
| ------ | ------ | | ------ | ------ |
| item_pgp_key:*item_path* | *key* | | item_pgpdump_key:*item_path* | *key* |
| | | | | |
| item_pgp_name:*item_path* | *name* | | item_pgpdump_name:*item_path* | *name* |
| | | | | |
| item_pgp_mail:*item_path* | *mail* | | item_pgpdump_mail:*item_path* | *mail* |
#### Cryptocurrency #### Cryptocurrency
@ -221,7 +221,7 @@ Supported cryptocurrency:
##### Hset date: ##### Hset date:
| Key | Field | Value | | Key | Field | Value |
| ------ | ------ | | ------ | ------ |
| cryptocurrency_**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** | | cryptocurrency:**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** |
##### zset: ##### zset:
| Key | Field | Value | | Key | Field | Value |

13
bin/Bitcoin.py
View file

@ -21,6 +21,7 @@ from pubsublogger import publisher
import re import re
import time import time
import redis
from hashlib import sha256 from hashlib import sha256
@ -58,7 +59,7 @@ def search_key(content, message, paste):
print('Bitcoin private key found : {}'.format(private_key)) print('Bitcoin private key found : {}'.format(private_key))
key = True key = True
# build bitcoin correlation # build bitcoin correlation
save_bitcoin_data(date, message, address): save_cryptocurrency_data('bitcoin', date, message, address)
if(validate_address): if(validate_address):
p.populate_set_out(message, 'Duplicate') p.populate_set_out(message, 'Duplicate')
@ -78,7 +79,7 @@ def search_key(content, message, paste):
publisher.warning('{}Detected {} Bitcoin private key;{}'.format( publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
to_print, len(bitcoin_private_key),paste.p_rel_path)) to_print, len(bitcoin_private_key),paste.p_rel_path))
def save_bitcoin_data(cryptocurrency_name, date, item_path, cryptocurrency_address): def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
# create basic medata # create basic medata
if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)): if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date) serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
@ -95,7 +96,7 @@ def save_bitcoin_data(cryptocurrency_name, date, item_path, cryptocurrency_addre
serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path) serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
# daily # daily
serv_metadata.hincrby('cryptocurrency_{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1) serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
# all type # all type
serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1) serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
@ -112,6 +113,12 @@ if __name__ == "__main__":
# Setup the I/O queues # Setup the I/O queues
p = Process(config_section) p = Process(config_section)
serv_metadata = redis.StrictRedis(
host=p.config.get("ARDB_Metadata", "host"),
port=p.config.getint("ARDB_Metadata", "port"),
db=p.config.getint("ARDB_Metadata", "db"),
decode_responses=True)
# Sent to the logging a description of the module # Sent to the logging a description of the module
publisher.info("Run Keys module ") publisher.info("Run Keys module ")

20
bin/PgpDump.py
View file

@ -90,28 +90,28 @@ def extract_id_from_output(pgp_dump_outpout):
def save_pgp_data(type_pgp, date, item_path, data): def save_pgp_data(type_pgp, date, item_path, data):
# create basic medata # create basic medata
if not serv_metadata.exists('pgp_{}:{}'.format(type_pgp, data)): if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date) serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else: else:
last_seen = serv_metadata.hget('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen') last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
if not last_seen: if not last_seen:
serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else: else:
if int(last_seen) < int(date): if int(last_seen) < int(date):
serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
# global set # global set
serv_metadata.sadd('pgp_{}:{}'.format(type_pgp, data), item_path) serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
# daily # daily
serv_metadata.hincrby('pgp:{}:{}'.format(type_pgp, date), data, 1) serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
# all type # all type
serv_metadata.zincrby('pgp_all:{}'.format(type_pgp), data, 1) serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
# item_metadata # item_metadata
serv_metadata.sadd('item_pgp_{}:{}'.format(type_pgp, item_path), data) serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
if __name__ == '__main__': if __name__ == '__main__':

778
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -31,6 +31,10 @@ PASTES_FOLDER = Flask_config.PASTES_FOLDER
hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates') hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
## TODO: put me in option
all_cryptocurrency = ['bitcoin', 'monero']
all_pgpdump = ['key', 'name', 'mail']
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def get_date_range(num_day): def get_date_range(num_day):
@ -62,15 +66,6 @@ def list_sparkline_values(date_range_sparkline, hash):
sparklines_value.append(int(nb_seen_this_day)) sparklines_value.append(int(nb_seen_this_day))
return sparklines_value return sparklines_value
def list_sparkline_pgp_values(date_range_sparkline, type_id, key_id):
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_metadata.hget('pgp:{}:{}'.format(type_id, date_day), key_id)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_file_icon(estimated_type): def get_file_icon(estimated_type):
file_type = estimated_type.split('/')[0] file_type = estimated_type.split('/')[0]
# set file icon # set file icon
@ -103,63 +98,98 @@ def get_file_icon_text(estimated_type):
return file_icon_text return file_icon_text
def get_pgp_id_icon_text(type_id): def get_icon(correlation_type, type_id):
icon_text = 'fas fa-sticky-note'
if correlation_type == 'pgpdump':
# set type_id icon # set type_id icon
if type_id == 'key': if type_id == 'key':
file_icon_text = '\uf084' icon_text = 'fas fa-key'
elif type_id == 'name': elif type_id == 'name':
file_icon_text = '\uf507' icon_text = 'fas fa-user-tag'
elif type_id == 'mail': elif type_id == 'mail':
file_icon_text = '\uf1fa' icon_text = 'fas fa-at'
else: else:
file_icon_text = '\uf249' icon_text = 'times'
return file_icon_text elif correlation_type == 'cryptocurrency':
if type_id == 'bitcoin':
icon_text = 'fab fa-btc'
elif type_id == 'monero':
icon_text = 'fab fa-monero'
elif type_id == 'ethereum':
icon_text = 'fab fa-ethereum'
else:
icon_text = 'fas fa-coins'
return icon_text
def get_pgp_icon(type_id): def get_icon_text(correlation_type, type_id):
# set type_id icon icon_text = '\uf249'
if correlation_type == 'pgpdump':
if type_id == 'key': if type_id == 'key':
pgp_icon_text = 'key' icon_text = '\uf084'
elif type_id == 'name': elif type_id == 'name':
pgp_icon_text = 'user-tag' icon_text = '\uf507'
elif type_id == 'mail': elif type_id == 'mail':
pgp_icon_text = 'at' icon_text = '\uf1fa'
else: else:
pgp_icon_text = 'times' icon_text = 'times'
return pgp_icon_text elif correlation_type == 'cryptocurrency':
if type_id == 'bitcoin':
icon_text = '\uf15a'
elif type_id == 'monero':
icon_text = '\uf3d0'
elif type_id == 'ethereum':
icon_text = '\uf42e'
else:
icon_text = '\uf51e'
return icon_text
def verify_pgp_type_id(type_id): def get_all_types_id(correlation_type):
if type_id in ['key', 'name', 'mail']: if correlation_type == 'pgpdump':
return all_pgpdump
elif correlation_type == 'cryptocurrency':
return all_cryptocurrency
else:
return []
def is_valid_type_id(correlation_type, type_id):
all_type_id = get_all_types_id(correlation_type)
if type_id in all_type_id:
return True return True
else: else:
return False return False
def get_key_id_metadata(type_id, key_id): def get_key_id_metadata(correlation_type, type_id, key_id):
key_id_metadata = {} key_id_metadata = {}
if r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)): if r_serv_metadata.exists('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id)):
key_id_metadata['first_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'first_seen') key_id_metadata['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id), 'first_seen')
key_id_metadata['first_seen'] = '{}/{}/{}'.format(key_id_metadata['first_seen'][0:4], key_id_metadata['first_seen'][4:6], key_id_metadata['first_seen'][6:8]) key_id_metadata['first_seen'] = '{}/{}/{}'.format(key_id_metadata['first_seen'][0:4], key_id_metadata['first_seen'][4:6], key_id_metadata['first_seen'][6:8])
key_id_metadata['last_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'last_seen') key_id_metadata['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id), 'last_seen')
key_id_metadata['last_seen'] = '{}/{}/{}'.format(key_id_metadata['last_seen'][0:4], key_id_metadata['last_seen'][4:6], key_id_metadata['last_seen'][6:8]) key_id_metadata['last_seen'] = '{}/{}/{}'.format(key_id_metadata['last_seen'][0:4], key_id_metadata['last_seen'][4:6], key_id_metadata['last_seen'][6:8])
key_id_metadata['nb_seen'] = r_serv_metadata.scard('pgp_{}:{}'.format(type_id, key_id)) key_id_metadata['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(correlation_type, type_id, key_id))
return key_id_metadata return key_id_metadata
def get_all_pgp_from_item(item_path): def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
all_pgp_dump = set() sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date_day), key_id)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_all_keys_id_from_item(correlation_type, item_path):
all_keys_id_dump = set()
if item_path is not None: if item_path is not None:
res = r_serv_metadata.smembers('item_pgp_key:{}'.format(item_path)) for type_id in get_all_types_id(correlation_type):
for pgp_dump in res: res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_type, type_id, item_path))
all_pgp_dump.add( (pgp_dump, 'key') ) for key_id in res:
res = r_serv_metadata.smembers('item_pgp_name:{}'.format(item_path)) all_keys_id_dump.add( (key_id, type_id) )
for pgp_dump in res: return all_keys_id_dump
all_pgp_dump.add( (pgp_dump, 'name') )
res = r_serv_metadata.smembers('item_pgp_mail:{}'.format(item_path))
for pgp_dump in res:
all_pgp_dump.add( (pgp_dump, 'mail') )
return all_pgp_dump
def one(): def one():
return 1 return 1
'''
def decode_base58(bc, length): def decode_base58(bc, length):
n = 0 n = 0
for char in bc: for char in bc:
@ -172,25 +202,275 @@ def check_bc(bc):
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
except Exception: except Exception:
return False return False
'''
def get_bitcoin_address_metadata(bitcoin_address): def get_correlation_type_search_endpoint(correlation_type):
address_metadata = {} if correlation_type == 'pgpdump':
if r_serv_metadata.exists('bitcoin_metadata:{}'.format(bitcoin_address)): endpoint = 'hashDecoded.all_pgpdump_search'
address_metadata['first_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'first_seen') elif correlation_type == 'cryptocurrency':
address_metadata['first_seen'] = '{}/{}/{}'.format(address_metadata['first_seen'][0:4], address_metadata['first_seen'][4:6], address_metadata['first_seen'][6:8]) endpoint = 'hashDecoded.all_cryptocurrency_search'
address_metadata['last_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'last_seen') else:
address_metadata['last_seen'] = '{}/{}/{}'.format(address_metadata['last_seen'][0:4], address_metadata['last_seen'][4:6], address_metadata['last_seen'][6:8]) endpoint = 'hashDecoded.hashDecoded_page'
address_metadata['nb_seen'] = r_serv_metadata.scard('bitcoin:{}'.format(bitcoin_address)) return endpoint
return address_metadata
def list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address): def get_correlation_type_page_endpoint(correlation_type):
sparklines_value = [] if correlation_type == 'pgpdump':
for date_day in date_range_sparkline: endpoint = 'hashDecoded.pgpdump_page'
nb_seen_this_day = r_serv_metadata.hget('bitcoin:{}'.format(bitcoin_address), bitcoin_address) elif correlation_type == 'cryptocurrency':
if nb_seen_this_day is None: endpoint = 'hashDecoded.cryptocurrency_page'
nb_seen_this_day = 0 else:
sparklines_value.append(int(nb_seen_this_day)) endpoint = 'hashDecoded.hashDecoded_page'
return sparklines_value return endpoint
def get_show_key_id_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.show_pgpdump'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.show_cryptocurrency'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_range_type_json_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.pgpdump_range_type_json'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.cryptocurrency_range_type_json'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_graph_node_json_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.pgpdump_graph_node_json'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.cryptocurrency_graph_node_json'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_graph_line_json_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.pgpdump_graph_line_json'
elif correlation_type == 'cryptocurrency':
endpoint = 'hashDecoded.cryptocurrency_graph_line_json'
else:
endpoint = 'hashDecoded.hashDecoded_page'
return endpoint
def get_font_family(correlation_type):
if correlation_type == 'pgpdump':
font = 'fa'
elif correlation_type == 'cryptocurrency':
font = 'fab'
else:
font = 'fa'
return font
############ CORE CORRELATION ############
def main_correlation_page(correlation_type, type_id, date_from, date_to, show_decoded_files):
if type_id == 'All types':
type_id = None
# verify type input
if type_id is not None:
#retrieve char
type_id = type_id.replace(' ', '')
if not is_valid_type_id(correlation_type, type_id):
type_id = None
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
# display day type bar chart
if len(date_range) == 1 and type is None:
daily_type_chart = True
daily_date = date_range[0]
else:
daily_type_chart = False
daily_date = None
if type_id is None:
all_type_id = get_all_types_id(correlation_type)
else:
all_type_id = type_id
l_keys_id_dump = set()
if show_decoded_files:
for date in date_range:
if isinstance(all_type_id, str):
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, all_type_id, date))
if l_dump:
for dump in l_dump:
l_keys_id_dump.add( (dump, all_type_id) )
else:
for typ_id in all_type_id:
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, typ_id, date))
if l_dump:
for dump in l_dump:
l_keys_id_dump.add( (dump, typ_id) )
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_id = 0
keys_id_metadata = {}
for dump_res in l_keys_id_dump:
new_key_id, typ_id = dump_res
keys_id_metadata[new_key_id] = get_key_id_metadata(correlation_type, typ_id, new_key_id)
if keys_id_metadata[new_key_id]:
keys_id_metadata[new_key_id]['type_id'] = typ_id
keys_id_metadata[new_key_id]['type_icon'] = get_icon(correlation_type, typ_id)
keys_id_metadata[new_key_id]['sparklines_data'] = list_sparkline_type_id_values(date_range_sparkline, correlation_type, typ_id, new_key_id)
keys_id_metadata[new_key_id]['sparklines_id'] = sparkline_id
sparkline_id += 1
l_type = get_all_types_id(correlation_type)
return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata,
correlation_type=correlation_type,
correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type),
correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type),
show_key_id_endpoint=get_show_key_id_endpoint(correlation_type),
range_type_json_endpoint=get_range_type_json_endpoint(correlation_type),
l_type=l_type, type_id=type_id,
daily_type_chart=daily_type_chart, daily_date=daily_date,
date_from=date_from, date_to=date_to,
show_decoded_files=show_decoded_files)
def show_correlation(correlation_type, type_id, key_id):
if is_valid_type_id(correlation_type, type_id):
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id)
if key_id_metadata:
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id)
return render_template('showCorrelation.html', key_id=key_id, type_id=type_id,
correlation_type=correlation_type,
graph_node_endpoint=get_graph_node_json_endpoint(correlation_type),
graph_line_endpoint=get_graph_line_json_endpoint(correlation_type),
font_family=get_font_family(correlation_type),
key_id_metadata=key_id_metadata,
type_icon=get_icon(correlation_type, type_id),
sparkline_values=sparkline_values)
else:
return '404'
else:
return 'error'
def correlation_type_range_type_json(correlation_type, date_from, date_to):
date_range = []
if date_from is not None and date_to is not None:
#change format
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
range_type = []
all_types_id = get_all_types_id(correlation_type)
# one day
if len(date_range) == 1:
for type_id in all_types_id:
day_type = {}
# init 0
for typ_id in all_types_id:
day_type[typ_id] = 0
day_type['date'] = type_id
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date_range[0]))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
#if day_type[type_id] != 0:
range_type.append(day_type)
else:
# display type_id
for date in date_range:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type_id in all_types_id:
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
range_type.append(day_type)
return jsonify(range_type)
def correlation_graph_node_json(correlation_type, type_id, key_id):
if key_id is not None and is_valid_type_id(correlation_type, type_id):
nodes_set_dump = set()
nodes_set_paste = set()
links_set = set()
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id)
nodes_set_dump.add((key_id, 1, type_id, key_id_metadata['first_seen'], key_id_metadata['last_seen'], key_id_metadata['nb_seen']))
#get related paste
l_pastes = r_serv_metadata.smembers('set_{}_{}:{}'.format(correlation_type, type_id, key_id))
for paste in l_pastes:
nodes_set_paste.add((paste, 2))
links_set.add((key_id, paste))
for key_id_with_type_id in get_all_keys_id_from_item(correlation_type, paste):
new_key_id, typ_id = key_id_with_type_id
if new_key_id != key_id:
key_id_metadata = get_key_id_metadata(correlation_type, typ_id, new_key_id)
nodes_set_dump.add((new_key_id, 3, typ_id, key_id_metadata['first_seen'], key_id_metadata['last_seen'], key_id_metadata['nb_seen']))
links_set.add((new_key_id, paste))
nodes = []
for node in nodes_set_dump:
nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_icon_text(correlation_type, node[2]),"url": url_for(get_show_key_id_endpoint(correlation_type), hash=node[0]), 'hash': True})
for node in nodes_set_paste:
nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False})
links = []
for link in links_set:
links.append({"source": link[0], "target": link[1]})
json = {"nodes": nodes, "links": links}
return jsonify(json)
else:
return jsonify({})
# ============= ROUTES ============== # ============= ROUTES ==============
@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST']) @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
@ -202,15 +482,6 @@ def all_hash_search():
show_decoded_files = request.form.get('show_decoded_files') show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files)) return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
@hashDecoded.route("/hashDecoded/all_pgp_dump_search", methods=['POST'])
def all_pgp_dump_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route("/hashDecoded/", methods=['GET']) @hashDecoded.route("/hashDecoded/", methods=['GET'])
def hashDecoded_page(): def hashDecoded_page():
date_from = request.args.get('date_from') date_from = request.args.get('date_from')
@ -813,253 +1084,6 @@ def update_vt_result():
############################ PGPDump ############################ ############################ PGPDump ############################
@hashDecoded.route("/decoded/pgpdump", methods=['GET'])
def pgpdump_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
show_decoded_files = request.args.get('show_decoded_files')
if type_id == 'All types':
type_id = None
# verify type input
if type_id is not None:
#retrieve char
type_id = type_id.replace(' ', '')
if not verify_pgp_type_id(type_id):
type_id = None
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
# display day type bar chart
if len(date_range) == 1 and type is None:
daily_type_chart = True
daily_date = date_range[0]
else:
daily_type_chart = False
daily_date = None
if type_id is None:
all_type_id = ['key', 'name', 'mail']
else:
all_type_id = type_id
l_pgp_dump = set()
if show_decoded_files:
for date in date_range:
if isinstance(all_type_id, str):
l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(all_type_id, date))
if l_dump:
for dump in l_dump:
l_pgp_dump.add( (dump, all_type_id) )
else:
for typ_id in all_type_id:
l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(typ_id, date))
if l_dump:
for dump in l_dump:
l_pgp_dump.add( (dump, typ_id) )
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_id = 0
pgp_metadata = {}
for dump_res in l_pgp_dump:
dump_id, typ_id = dump_res
pgp_metadata[dump_id] = get_key_id_metadata(typ_id, dump_id)
if pgp_metadata[dump_id]:
pgp_metadata[dump_id]['type_id'] = typ_id
pgp_metadata[dump_id]['type_icon'] = get_pgp_icon(typ_id)
pgp_metadata[dump_id]['sparklines_data'] = list_sparkline_pgp_values(date_range_sparkline, typ_id, dump_id)
pgp_metadata[dump_id]['sparklines_id'] = sparkline_id
sparkline_id += 1
l_type = ['key', 'name', 'mail']
return render_template("PgpDecoded.html", l_pgpdump=pgp_metadata,
l_type=l_type, type_id=type_id, daily_type_chart=daily_type_chart, daily_date=daily_date,
date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files)
@hashDecoded.route('/decoded/show_pgpdump')
def show_pgpdump():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
if verify_pgp_type_id(type_id):
key_id_metadata = get_key_id_metadata(type_id, key_id)
if key_id_metadata:
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_values = list_sparkline_pgp_values(date_range_sparkline, type_id, key_id)
return render_template('showPgpDump.html', key_id=key_id, type_id=type_id,
key_id_metadata=key_id_metadata,
sparkline_values=sparkline_values)
else:
return '404'
else:
return 'error'
@hashDecoded.route('/decoded/pgp_dump_graph_node_json')
def pgp_dump_graph_node_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
if key_id is not None and verify_pgp_type_id(type_id):
nodes_set_pgp_dump = set()
nodes_set_paste = set()
links_set = set()
pgp_id_metadata = get_key_id_metadata(type_id, key_id)
#key_id = '</b><script>alert("Hello! I am an alert box!!");</script>'
nodes_set_pgp_dump.add((key_id, 1, type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen']))
#get related paste
l_pastes = r_serv_metadata.smembers('pgp_{}:{}'.format(type_id, key_id))
for paste in l_pastes:
nodes_set_paste.add((paste, 2))
links_set.add((key_id, paste))
l_pgpdump = get_all_pgp_from_item(paste)
for pgp_dump_with_type in l_pgpdump:
pgp_dump, pgp_type_id = pgp_dump_with_type
if pgp_dump != key_id:
pgp_id_metadata = get_key_id_metadata(pgp_type_id, pgp_dump)
nodes_set_pgp_dump.add((pgp_dump, 3, pgp_type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen']))
links_set.add((pgp_dump, paste))
nodes = []
for node in nodes_set_pgp_dump:
nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_pgp_id_icon_text(node[2]),"url": url_for('hashDecoded.showHash', hash=node[0]), 'hash': True})
for node in nodes_set_paste:
nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False})
links = []
for link in links_set:
links.append({"source": link[0], "target": link[1]})
json = {"nodes": nodes, "links": links}
return jsonify(json)
else:
return jsonify({})
@hashDecoded.route('/hashDecoded/pgp_graph_line_json')
def pgp_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
# verify input
if key_id is not None and verify_pgp_type_id(type_id) and r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)):
if date_from is None or date_to is None:
nb_days_seen_in_pastes = 30
else:
# # TODO: # FIXME:
nb_days_seen_in_pastes = 30
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
json_seen_in_paste = []
for date in date_range_seen_in_pastes:
nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash)
if nb_seen_this_day is None:
nb_seen_this_day = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
return jsonify(json_seen_in_paste)
else:
return jsonify()
@hashDecoded.route('/decoded/pgp_range_type_json')
def pgp_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
date_range = []
if date_from is not None and date_to is not None:
#change format
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
range_type = []
list_type_id = ['key', 'name', 'mail']
if len(date_range) == 1:
for type_id in list_type_id:
day_type = {}
day_type['key'] = 0
day_type['name'] = 0
day_type['mail'] = 0
day_type['date'] = type_id
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date_range[0]))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
#if day_type[type_id] != 0:
range_type.append(day_type)
else:
# display type_id
for date in date_range:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type_id in list_type_id:
num_day_type_id = 0
all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date))
if all_keys:
for val in all_keys:
num_day_type_id += int(val)
day_type[type_id]= num_day_type_id
range_type.append(day_type)
return jsonify(range_type)
@hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR @hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR
def pgp_by_type_json(): def pgp_by_type_json():
type_id = request.args.get('type_id') type_id = request.args.get('type_id')
@ -1103,29 +1127,119 @@ def pgp_by_type_json():
else: else:
return jsonify() return jsonify()
############################ Bitcoin ############################ ############################ Correlation ############################
''' @hashDecoded.route("/correlation/pgpdump", methods=['GET'])
@hashDecoded.route('/correlation/show_bitcoin_address') def pgpdump_page():
def show_bitcoin_address(): date_from = request.args.get('date_from')
bitcoin_address = request.args.get('bitcoin_address') date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
# validate user input show_decoded_files = request.args.get('show_decoded_files')
if check_bc(bitcoin_address): res = main_correlation_page('pgpdump', type_id, date_from, date_to, show_decoded_files)
bitcoin_address_metadata = get_bitcoin_address_metadata(bitcoin_address) return res
if bitcoin_address_metadata:
num_day_sparkline = 6 @hashDecoded.route("/correlation/cryptocurrency", methods=['GET'])
date_range_sparkline = get_date_range(num_day_sparkline) def cryptocurrency_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type_id = request.args.get('type_id')
sparkline_values = list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address) show_decoded_files = request.args.get('show_decoded_files')
return render_template('showPgpDump.html', bitcoin_address=bitcoin_address, res = main_correlation_page('cryptocurrency', type_id, date_from, date_to, show_decoded_files)
key_id_metadata=bitcoin_address_metadata, return res
sparkline_values=sparkline_values)
@hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST'])
def all_pgpdump_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST'])
def all_cryptocurrency_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type_id = request.form.get('type')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
@hashDecoded.route('/correlation/show_pgpdump')
def show_pgpdump():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
return show_correlation('pgpdump', type_id, key_id)
@hashDecoded.route('/correlation/show_cryptocurrency')
def show_cryptocurrency():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
return show_correlation('cryptocurrency', type_id, key_id)
@hashDecoded.route('/correlation/cryptocurrency_range_type_json')
def cryptocurrency_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_type_range_type_json('cryptocurrency', date_from, date_to)
@hashDecoded.route('/correlation/pgpdump_range_type_json')
def pgpdump_range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_type_range_type_json('pgpdump', date_from, date_to)
@hashDecoded.route('/correlation/pgpdump_graph_node_json')
def pgpdump_graph_node_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
return correlation_graph_node_json('pgpdump', type_id, key_id)
@hashDecoded.route('/correlation/cryptocurrency_graph_node_json')
def cryptocurrency_graph_node_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
return correlation_graph_node_json('cryptocurrency', type_id, key_id)
@hashDecoded.route('/correlation/pgpdump_graph_line_json')
def pgpdump_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_graph_line_json('pgpdump', type_id, key_id, date_from, date_to)
def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, date_to):
# verify input
if key_id is not None and is_valid_type_id(correlation_type, type_id) and r_serv_metadata.exists('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id)):
if date_from is None or date_to is None:
nb_days_seen_in_pastes = 30
else: else:
return '404' # # TODO: # FIXME:
nb_days_seen_in_pastes = 30
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
json_seen_in_paste = []
for date in date_range_seen_in_pastes:
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date), key_id)
if nb_seen_this_day is None:
nb_seen_this_day = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
return jsonify(json_seen_in_paste)
else: else:
return 'error' return jsonify()
'''
@hashDecoded.route('/correlation/cryptocurrency_graph_line_json')
def cryptocurrency_graph_line_json():
type_id = request.args.get('type_id')
key_id = request.args.get('key_id')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
return correlation_graph_line_json('cryptocurrency', type_id, key_id, date_from, date_to)
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(hashDecoded, url_prefix=baseUrl) app.register_blueprint(hashDecoded, url_prefix=baseUrl)

660
var/www/modules/hashDecoded/templates/DaysCorrelation.html Normal file
View file

@ -0,0 +1,660 @@
<!DOCTYPE html>
<html>
<head>
<title>Decoded - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.input-group .form-control {
position: unset;
}
.line {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.bar {
fill: steelblue;
}
.bar:hover{
fill: brown;
cursor: pointer;
}
.bar_stack:hover{
cursor: pointer;
}
.pie_path:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'decoded/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="row">
<div class="col-xl-10">
<div class="mt-1" id="barchart_type">
</div>
</div>
<div class="col-xl-2">
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
<div class="card-body text-center py-2">
<h6 class="card-title" style="color:#286090;">Select a date range :</h5>
<form action="{{ url_for(correlation_type_search_endpoint) }}" id="hash_selector_form" method='post'>
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
</div>
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
</div>
<div class="mt-1" style="font-size: 14px;color:#286090;">Type ID :</div>
<select class="custom-select" name="type">
<option>All types</option>
{% for typ in l_type %}
{% if type_id|string() == typ|string() %}
<option selected>{{ typ }}</option>
{% else %}
<option>{{ typ }}</option>
{% endif %}
{% endfor %}
</select>
<div class="form-check my-1">
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
<label class="form-check-label" for="checkbox-input-show">
<div style="color:#286090; font-size: 14px;">
Show {{correlation_type}} <i class="fas fa-key"></i>
</div>
</label>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fas fa-copy"></i> Search
</button>
<form>
</div>
</div>
<div id="pie_chart_encoded">
</div>
<div id="pie_chart_top5_types">
</div>
</div>
</div>
{% if all_metadata|length != 0 %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }} {{correlation_type}}: </h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }} {{correlation_type}}: </h3>
{% endif %}
<table id="tableb64" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>type id</th>
<th>key id</th>
<th>first seen</th>
<th>last seen</th>
<th>nb paste</th>
<th>Sparkline</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for key_id in all_metadata %}
<tr>
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td>
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type_id={{ all_metadata[key_id]['type_id'] }}&key_id={{ key_id }}">{{ key_id }}</a></td>
<td>{{ all_metadata[key_id]['first_seen'] }}</td>
<td>{{ all_metadata[key_id]['last_seen'] }}</td>
<td>{{ all_metadata[key_id]['nb_seen'] }}</td>
<td id="sparklines_{{ all_metadata[key_id]['sparklines_id'] }}" style="text-align:center;"></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
{% if show_decoded_files %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No Dumped Keys</h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }}, No {{correlation_type}}</h3>
{% endif %}
{% endif %}
{% endif %}
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_dashboard_{{correlation_type}}").addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
},
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
},
});
$('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
$('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
$('#tableb64').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
});
{% if type_id %}
//chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&type_id={{type_id}}", 'id');
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id={{type_id}}", 'id');
{% elif type_id or daily_type_chart %}
chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
{% else %}
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
{% endif %}
//draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
//draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
chart.onResize();
$(window).on("resize", function() {
chart.onResize();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
// a sparklines plot
function sparklines(id, points) {
var width = 100, height = 60;
var data = []
for (i = 0; i < points.length; i++) {
data[i] = {
'x': i,
'y': +points[i]
}
}
var x = d3.scaleLinear()
.range([0, width - 10])
.domain([0,5]);
var y = d3.scaleLinear()
.range([height, 0])
.domain([0,10]);
var line = d3.line()
.x(function(d) {return x(d.x)})
.y(function(d) {return y(d.y)});
d3.select("#"+id).append('svg')
.attr('width', width)
.attr('height', height)
.append('path')
.attr('class','line')
.datum(data)
.attr('d', line);
}
</script>
<script>
{% for key_id in all_metadata %}
sparklines("sparklines_{{ all_metadata[key_id]['sparklines_id'] }}", {{ all_metadata[key_id]['sparklines_data'] }})
{% endfor %}
</script>
<script>
var margin = {top: 20, right: 100, bottom: 55, left: 45},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type_stack(url, id) {
d3.json(url)
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("class", "bar")
{% if date_from|string == date_to|string and type is none %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
.attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-18)" )
{% else %}
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-40)" )
{% endif %}
.style("text-anchor", "end");
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and type is none %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
{% else %}
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
{% endif %}
data.forEach(function(d) {
if(d.total != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total))
{% if date_from|string == date_to|string and type is none %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
{% else %}
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from='+d.date+'&date_to='+d.date })
{% endif %}
.style("text-anchor", "middle")
.text(d.total);
}
});
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: "<b><span id='tooltip-id-name-bar'></span></b>",
placement: 'top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "<span id='tooltip-id-label'></span>" +
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
});
$(this).popover('show');
$("#tooltip-id-name-bar").text(d.name);
$("#tooltip-id-label").text(d.label);
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
}
chart.onResize = function () {
var aspect = 1000 / 500, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.chart = chart;
</script>
<script>
function draw_pie_chart(id, url_json, pie_on_click_url) {
var width_pie = 200;
var height_pie = 200;
var padding_pie = 10;
var opacity_pie = .8;
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
var div_pie = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var svg_pie = d3.select("#"+id)
.append('svg')
.attr("width", '100%')
.attr("height", '100%')
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
.attr('preserveAspectRatio','xMinYMin')
var g_pie = svg_pie.append('g')
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
var arc_pie = d3.arc()
.innerRadius(0)
.outerRadius(radius_pie);
d3.json(url_json)
.then(function(data){
var pie_pie = d3.pie()
.value(function(d) { return d.value; })
.sort(null);
var path_pie = g_pie.selectAll('path')
.data(pie_pie(data))
.enter()
.append("g")
.append('path')
.attr('d', arc_pie)
.attr('fill', (d,i) => color_pie(i))
.attr('class', 'pie_path')
.on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
.style('opacity', opacity_pie)
.style('stroke', 'white');
});
function mouseovered_pie(d) {
//remove old content
$("#tooltip-id-name").remove();
$("#tooltip-id-value").remove();
// tooltip
var content;
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
div_pie.transition()
.duration(200)
.style("opacity", .9);
div_pie.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.data.name);
$("#tooltip-id-value").text(d.data.value);
}
function mouseouted_pie() {
div_pie.transition()
.duration(500)
.style("opacity", 0);
}
}
</script>
<script>
function barchart_type(url, id) {
var margin = {top: 20, right: 20, bottom: 70, left: 40};
var width = 960 - margin.left - margin.right;
var height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x)
//.tickFormat(d3.time.format("%Y-%m"));
var yAxis = d3.axisLeft(y)
.ticks(10);
/*var svg = d3.select(id).append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.attr("id", "thesvg")
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");*/
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.value = +d.value;
});
x.domain(data.map(function(d) { return d.date; }));
y.domain([0, d3.max(data, function(d) { return d.value; })]);
var label = svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", "-.55em")
{% if daily_type_chart %}
.attr("transform", "rotate(-20)" );
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d });
{% endif %}
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Value ($)");
var bar = svg.selectAll("bar")
.data(data)
.enter().append("rect")
.attr("class", "bar")
//.style("fill", "steelblue")
.attr("x", function(d) { return x(d.date); })
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
{% if type %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
{% endif %}
{% if daily_type_chart %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
{% endif %}
data.forEach(function(d) {
if(d.value != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
//.text(function(d) { return d.value; });
.text(d.value)
.style("text-anchor", "middle")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.value));
}
});
});
}
</script>
</body>
</html>

553
var/www/modules/hashDecoded/templates/showCorrelation.html Normal file
View file

@ -0,0 +1,553 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>AIL - framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
line.link {
stroke: #666;
}
line.link:hover{
stroke: red;
stroke-width: 2px
}
.line_sparkline {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.node {
pointer-events: all;
}
circle {
stroke: none;
}
.graph_text_node {
font: 8px sans-serif;
pointer-events: none;
}
.graph_node_icon {
pointer-events: none;
}
.node text {
font: 8px sans-serif;
pointer-events: auto;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
.graph_panel {
padding: unset;
}
.line_graph {
fill: none;
stroke: steelblue;
stroke-width: 2px;
stroke-linejoin: round;
stroke-linecap: round;
stroke-width: 1.5;
/*attr('stroke', '#bcbd22').*/
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'decoded/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ key_id }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>type</th>
<th>First_seen</th>
<th>Last_seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td><i class="{{ type_icon }}"></i>&nbsp;&nbsp;{{ type_id }}</td>
<td>{{ key_id_metadata['first_seen'] }}</td>
<td>{{ key_id_metadata['last_seen'] }}</td>
<td>{{ key_id_metadata['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
</div>
</div>
<div class="row">
<div class="col-xl-10">
<div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body graph_panel">
<div id="graph">
</div>
</div>
</div>
</div>
<div class="col-xl-2">
<div class="card my-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
</div>
<div class="card-body text-center px-0 py-0">
<button class="btn btn-primary my-4" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
{% if correlation_type=='pgpdump' %}
{% include 'decoded/show_helper_pgpdump.html' %}
{% elif correlation_type=='cryptocurrency' %}
{% include 'decoded/show_helper_cryptocurrency.html' %}
{% endif %}
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-header">
<i class="fas fa-chart-bar"></i> Graph
</div>
<div class="panel-body ">
<div id="graph_line">
</div>
</div>
</div>
</div>
</div>
</div>
<script>
var all_graph = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
sparklines("sparkline", {{ sparkline_values }})
all_graph.node_graph = create_graph("{{ url_for(graph_node_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for(graph_line_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
all_graph.onResize();
});
$(window).on("resize", function() {
all_graph.onResize();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function resize_graph() {
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
}
</script>
<script>
//var data = [6,3,3,2,5,3,9];
// a sparklines plot
function sparklines(id, points) {
var width_spark = 100, height_spark = 60;
var data = []
for (i = 0; i < points.length; i++) {
data[i] = {
'x': i,
'y': +points[i]
}
}
var x = d3.scaleLinear()
.range([0, width_spark - 10])
.domain([0,5]);
var y = d3.scaleLinear()
.range([height_spark, 0])
.domain([0,10]);
var line = d3.line()
.x(function(d) {return x(d.x)})
.y(function(d) {return y(d.y)});
d3.select("#"+id).append('svg')
.attr('width', width_spark)
.attr('height', height_spark)
.append('path')
.attr('class','line_sparkline')
.datum(data)
.attr('d', line);
}
</script>
<script>
var width = 400,
height = 400;
var link;
var zoom = d3.zoom()
.scaleExtent([.2, 10])
.on("zoom", zoomed);
//var transform = d3.zoomIdentity;
var color = d3.scaleOrdinal(d3.schemeCategory10);
var div = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) { return d.id; }))
.force("charge", d3.forceManyBody())
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
.attr("height", height)
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
.on("dblclick.zoom", null)
var container_graph = svg_node.append("g");
//.attr("transform", "translate(40,0)")
//.attr("transform", "scale(2)");
function create_graph(url){
d3.json(url)
.then(function(data){
link = container_graph.append("g")
.selectAll("line")
.data(data.links)
.enter().append("line")
.attr("class", "link");
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
var node = container_graph.selectAll(".node")
.data(data.nodes)
.enter().append("g")
.attr("class", "nodes")
.on("dblclick", doubleclick)
.on("click", click)
.on("mouseover", mouseovered)
.on("mouseout", mouseouted)
.call(d3.drag()
.on("start", drag_start)
.on("drag", dragged)
.on("end", drag_end));
node.append("circle")
.attr("r", function(d) {
return (d.hash) ? 6 : 5; })
.attr("fill", function(d) {
if(!d.hash){ return color(d.group);}
if(d.group == 1){ return "orange";}
return "rgb(141, 211, 199)"; });
node.append('text')
.attr('text-anchor', 'middle')
.attr('dominant-baseline', 'central')
.attr("class", "graph_node_icon {{font_family}}")
.attr('font-size', '8px' )
.attr('pointer-events', 'none')
.text(function(d) {
if(d.hash){
return d.icon
} });
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
/* node.append("text")
.attr("dy", 3)
.attr("dx", 7)
.attr("class", "graph_text_node")
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.id; });*/
simulation
.nodes(data.nodes)
.on("tick", ticked);
simulation.force("link")
.links(data.links);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
/*node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });*/
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
}
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
}
function doubleclick (d) {
window.open(d.url, '_blank');
}
function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
// tooltip
var content;
if(d.hash == true){
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>First seen</i>: <span id='tooltip-id-first_seen'></span><br/>"+
"<i>Last seen</i>: <span id='tooltip-id-last_seen'></span><br/>"+
"<i>nb_seen</i>: <span id='tooltip-id-nb_seen'></span>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
$("#tooltip-id-first_seen").text(d.first_seen);
$("#tooltip-id-last_seen").text(d.last_seen);
$("#tooltip-id-nb_seen").text(d.nb_seen_in_paste);
} else {
content = "<b><span id='tooltip-id-name'></span></b><br/>";
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.id);
}
//links
/*link.style("stroke-opacity", function(o) {
return o.source === d || o.target === d ? 1 : opacity;
});*/
link.style("stroke", function(o){
return o.source === d || o.target === d ? "#666" : "#ddd";
});
}
function mouseouted() {
div.transition()
.duration(500)
.style("opacity", 0);
link.style("stroke", "#666");
}
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>
</body>
</html>

20
var/www/templates/decoded/show_helper_cryptocurrency.html Normal file
View file

@ -0,0 +1,20 @@
<ul class="list-group">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
<li class="list-group-item text-left">
<p>Double click on a node to open Cryptocurrency/Paste<br><br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
Current Cryptocurrency<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
Cryptocurrency<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
Pastes
</p>
</li>
<li class="list-group-item list-group-item-info">
Cryptocurrency Types:
</li>
<li class="list-group-item text-left">
<i class="fab fa-btc"></i> Bitcoin<br>
<i class="fab fa-monero"></i> Monero<br>
</li>
</ul>

21
var/www/templates/decoded/show_helper_pgpdump.html Normal file
View file

@ -0,0 +1,21 @@
<ul class="list-group">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
<li class="list-group-item text-left">
<p>Double click on a node to open PgpDump/Paste<br><br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
Current PgpDump<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
PgpDump<br>
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
Pastes
</p>
</li>
<li class="list-group-item list-group-item-info">
PgpDump Types:
</li>
<li class="list-group-item text-left">
<i class="fas fa-key"></i> Key<br>
<i class="fas fa-user-tag"></i> Name<br>
<i class="fas fa-at"></i> Mail<br>
</li>
</ul>