chg: [decoded migration] migrate Decoded Objects

This commit is contained in:
Terrtia 2023-01-09 16:03:06 +01:00
parent b5d285b5b4
commit d295c084c6
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
20 changed files with 1210 additions and 727 deletions

View file

@ -612,8 +612,10 @@ def domain_migration():
def get_estimated_type(decoded_id):
return r_serv_metadata.hget(f'metadata_hash:{decoded_id}', 'estimated_type')
def get_decoded_items_list_by_decoder(decoder_type, decoded_id): ###################
#return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1)
def get_hash_size(decoded_id):
return r_serv_metadata.hget(f'metadata_hash:{decoded_id}', 'size')
def get_decoded_items_list_by_decoder(decoder_type, decoded_id):
return r_serv_metadata.zrange(f'{decoder_type}_hash:{decoded_id}', 0, -1)
def get_decodeds_tags(decoded_id):
@ -621,10 +623,10 @@ def get_decodeds_tags(decoded_id):
def decodeds_migration():
print('Decoded MIGRATION...')
decoder_names = ['base64', 'binary', 'hexadecimal']
algo_names = ['base64', 'binary', 'hexadecimal']
Decodeds._delete_old_json_descriptor()
for decoded_id in Decodeds.get_all_decodeds():
for decoded_id in Decodeds.get_all_decodeds_files():
mimetype = get_estimated_type(decoded_id)
# ignore invalid object
if mimetype is None:
@ -633,19 +635,23 @@ def decodeds_migration():
print(decoded_id)
decoded = Decodeds.Decoded(decoded_id)
filepath = decoded.get_filepath(mimetype=mimetype)
decoded._save_meta(filepath, mimetype)
decoded._add_create()
decoded.set_mimetype(mimetype)
size = get_hash_size(decoded_id)
if not size:
filepath = decoded.get_filepath(mimetype=mimetype)
size = os.path.getsize(filepath)
decoded.set_size(size)
for tag in get_decodeds_tags(decoded_id):
decoded.add_tag(tag)
for decoder_type in decoder_names:
for item_id in get_decoded_items_list_by_decoder(decoder_type, decoded_id):
print(item_id, decoder_type)
for algo in algo_names:
for item_id in get_decoded_items_list_by_decoder(algo, decoded_id):
print(item_id, algo)
date = get_item_date(item_id)
#for decoder_type in :
decoded.add(decoder_type, date, item_id, mimetype)
decoded.add(algo, date, item_id, mimetype=mimetype)
###############################
# #
@ -868,7 +874,7 @@ if __name__ == '__main__':
# items_migration()
#crawler_migration()
# domain_migration() # TO TEST ###########################
#decodeds_migration()
decodeds_migration()
# screenshots_migration()
# subtypes_obj_migration()
# ail_2_ail_migration()
@ -876,7 +882,7 @@ if __name__ == '__main__':
# investigations_migration()
# statistics_migration()
cves_migration()
# cves_migration()
# custom tags
# crawler queues + auto_crawlers

View file

@ -96,7 +96,10 @@ def is_obj_correlated(obj_type, subtype, obj_id, obj2_type, subtype2, obj2_id):
subtype = ''
if subtype2 is None:
subtype2 = ''
return r_metadata.sismember(f'correlation:obj:{obj_type}:{subtype}:{obj2_type}:{obj_id}', f'{subtype2}:{obj2_id}')
try:
return r_metadata.sismember(f'correlation:obj:{obj_type}:{subtype}:{obj2_type}:{obj_id}', f'{subtype2}:{obj2_id}')
except:
return False
def add_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, obj2_id):
if subtype1 is None:

View file

@ -27,7 +27,7 @@ config_loader = None
# DOMAIN -> subtype = domain type
# TAG -> type = "TAG"
# TAG -> subtype = "OBJ:"tag
# TAG -> subtype = f"OBJ:{tag}"
def load_obj_date_first_last():
# LOAD FIRST DATE

View file

@ -84,7 +84,7 @@ class Cve(AbstractDaterangeObject):
# 'Modified'
return json_response
else:
return {'error': 'cve search error'} # TODO
return {'error': f'{response.status_code}'}
# TODO ADD SEARCH FUNCTION
@ -122,5 +122,14 @@ def api_get_cves_meta_by_daterange(date_from, date_to):
date = Date.sanitise_date_range(date_from, date_to)
return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline'])
def get_cve_graphline(cve_id):
cve = Cve(cve_id)
graphline = []
if cve.exists():
nb_day = 30
for date in Date.get_previous_date_list(nb_day):
graphline.append({'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', 'value': cve.get_nb_seen_by_date(date)})
return graphline
# if __name__ == '__main__':

View file

@ -8,12 +8,14 @@ import zipfile
from flask import url_for
from io import BytesIO
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_object import AbstractObject
from lib.item_basic import is_crawled, get_item_domain
from lib.objects.abstract_daterange_object import AbstractDaterangeObject
from packages import Date
sys.path.append('../../configs/keys')
@ -22,17 +24,17 @@ try:
if vt_key != '':
VT_TOKEN = vt_key
VT_ENABLED = True
#print('VT submission is enabled')
# print('VT submission is enabled')
else:
VT_ENABLED = False
#print('VT submission is disabled')
# print('VT submission is disabled')
except:
VT_TOKEN = None
VT_ENABLED = False
#print('VT submission is disabled')
# print('VT submission is disabled')
config_loader = ConfigLoader()
r_metadata = config_loader.get_db_conn("Kvrocks_Objects")
r_objects = config_loader.get_db_conn("Kvrocks_Objects")
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
@ -46,7 +48,7 @@ config_loader = None
# # TODO: COMPLETE CLASS
class Decoded(AbstractObject):
class Decoded(AbstractDaterangeObject):
"""
AIL Decoded Object. (strings)
"""
@ -54,9 +56,6 @@ class Decoded(AbstractObject):
def __init__(self, id):
super(Decoded, self).__init__('decoded', id)
def exists(self):
return r_metadata.exists(f'metadata_hash:{self.id}')
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
@ -74,9 +73,10 @@ class Decoded(AbstractObject):
url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
return url
def get_svg_icon(self):
file_type = self.get_estimated_type()
file_type = file_type.split('/')[0]
def get_svg_icon(self, mimetype=None):
if not mimetype:
mimetype = self.get_mimetype()
file_type = mimetype.split('/')[0]
if file_type == 'application':
icon = '\uf15b'
elif file_type == 'audio':
@ -87,19 +87,28 @@ class Decoded(AbstractObject):
icon = '\uf15c'
else:
icon = '\uf249'
return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5}
return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius': 5}
'''
Return the estimated type of a given decoded item.
:param sha1_string: sha1_string
'''
def get_estimated_type(self):
return r_metadata.hget(f'metadata_hash:{self.id}', 'estimated_type')
def get_mimetype(self):
return r_objects.hget(f'meta:{self.type}:{self.id}', 'mime')
def set_mimetype(self, mimetype):
return r_objects.hset(f'meta:{self.type}:{self.id}', 'mime', mimetype)
def get_size(self):
return r_objects.hget(f'meta:{self.type}:{self.id}', 'size')
def set_size(self, size):
return r_objects.hset(f'meta:{self.type}:{self.id}', 'size', int(size))
def get_rel_path(self, mimetype=None):
if not mimetype:
mimetype = self.get_estimated_type()
mimetype = self.get_mimetype()
return os.path.join(HASH_DIR, mimetype, self.id[0:2], self.id)
def get_filepath(self, mimetype=None):
@ -112,25 +121,24 @@ class Decoded(AbstractObject):
return file_content
def get_zip_content(self):
mimetype = self.get_estimated_type()
# mimetype = self.get_estimated_type()
zip_content = BytesIO()
with zipfile.ZipFile(zip_content, "w") as zf:
# TODO: Fix password
#zf.setpassword(b"infected")
zf.writestr( self.id, self.get_content().getvalue())
# zf.setpassword(b"infected")
zf.writestr(self.id, self.get_content().getvalue())
zip_content.seek(0)
return zip_content
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('file')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
obj_attrs.append( obj.add_attribute('sha1', value=self.id) )
obj_attrs.append( obj.add_attribute('mimetype', value=self.get_estimated_type()) )
obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()) )
obj_attrs.append( obj.add_attribute('sha1', value=self.id))
obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype()))
obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
@ -139,176 +147,84 @@ class Decoded(AbstractObject):
############################################################################
############################################################################
############################################################################
def get_decoders(self):
return ['base64', 'binary', 'hexadecimal']
def get_first_seen(self):
res = r_metadata.hget(f'metadata_hash:{self.id}', 'first_seen')
if res:
return int(res)
else:
return 99999999
def get_last_seen(self):
res = r_metadata.hget(f'metadata_hash:{self.id}', 'last_seen')
if res:
return int(res)
else:
return 0
def set_first_seen(self, date):
r_metadata.hset(f'metadata_hash:{self.id}', 'first_seen', date)
def set_last_seen(self, date):
r_metadata.hset(f'metadata_hash:{self.id}', 'last_seen', date)
def update_daterange(self, date):
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if date < first_seen:
self.set_first_seen(date)
if date > last_seen:
self.set_last_seen(date)
def get_meta(self, options=set()):
meta = {'id': self.id,
'subtype': self.subtype,
'tags': self.get_tags()}
meta = self._get_meta(options=options)
meta['id'] = self.id
if 'mimetype' in options:
meta['mimetype'] = self.get_mimetype()
if 'icon' in options:
if 'mimetype' in meta:
mimetype = meta['mimetype']
else:
mimetype = None
meta['icon'] = self.get_svg_icon(mimetype=mimetype)
if 'size' in options:
meta['size'] = self.get_size()
if 'tags' in options:
meta['tags'] = self.get_tags()
if 'vt' in options:
meta['vt'] = self.get_meta_vt()
return meta
def get_meta_vt(self):
meta = {}
meta['link'] = r_metadata.hget(f'metadata_hash:{self.id}', 'vt_link')
meta['report'] = r_metadata.hget(f'metadata_hash:{self.id}', 'vt_report')
return meta
link = r_objects.hget(f'meta:{self.type}:{self.id}', 'vt_link')
report = r_objects.hget(f'meta:{self.type}:{self.id}', 'vt_report')
if link or report:
return {'link': link, 'report': report}
else:
return {}
def guess_mimetype(self, bytes_content):
return magic.from_buffer(bytes_content, mime=True)
def _save_meta(self, filepath, mimetype):
# create hash metadata
r_metadata.hset(f'metadata_hash:{self.id}', 'size', os.path.getsize(filepath))
r_metadata.hset(f'metadata_hash:{self.id}', 'estimated_type', mimetype)
r_metadata.sadd('hash_all_type', mimetype) #################################################### rename ????
# avoid counting the same hash multiple time on the same item
# except if different encoding
def save_file(self, content, mimetype): #####################################################
def is_seen_this_day(self, date):
return bool(self.get_nb_seen_by_date(Date.get_today_date_str()))
def save_file(self, b_content, mimetype): # TODO TEST ME
filepath = self.get_filepath(mimetype=mimetype)
if os.path.isfile(filepath):
#print('File already exist')
# print('File already exist')
return False
# create dir
dirname = os.path.dirname(filepath)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filepath, 'wb') as f:
f.write(file_content)
f.write(b_content)
# create meta
self._add_create()
self.set_mimetype(mimetype)
self.set_size(os.path.getsize(filepath))
# create hash metadata
self._save_meta(filepath, mimetype)
return True
# avoid counting the same hash multiple time on the same item
# except if defferent encoding
def add(self, algo_name, date, obj_id, mimetype=None):
self._add(date, obj_id)
if not mimetype:
mimetype = self.get_mimetype()
def is_seen_this_day(self, date):
for decoder in get_decoders_names():
if r_metadata.zscore(f'{decoder}_date:{date}', self.id):
return True
return False
is_new_decoded = r_objects.sadd(f'decoded:algo:{algo_name}:{date}', self.id) # filter by algo + sparkline
# uniq decoded in this algo today
if int(is_new_decoded):
r_objects.zincrby(f'decoded:algos:{date}', 1, algo_name) # pie chart
def add(self, decoder_name, date, obj_id, mimetype):
# mimetype -> decodeds
is_new_decoded = r_objects.sadd(f'decoded:mime:{mimetype}:{date}', self.id) # filter by mimetype
# uniq decoded in this mimetype today
if int(is_new_decoded):
r_objects.zincrby(f'decoded:mime:{date}', 1, mimetype) # TDO ADD OPTION TO CALC IF NOT EXISTS
r_objects.sadd('decoded:mimetypes', mimetype)
if not self.is_seen_this_day(date):
# mimetype
r_metadata.zincrby(f'decoded:mimetype:{date}', 1, mimetype)
r_metadata.sadd(f'decoded:mimetypes', mimetype)
# filter hash encoded in the same object
if not self.is_correlated('item', None, obj_id):
r_metadata.hincrby(f'metadata_hash:{self.id}', f'{decoder_name}_decoder', 1)
r_metadata.zincrby(f'{decoder_name}_type:{mimetype}', 1, date)
r_metadata.incrby(f'{decoder_name}_decoded:{date}', 1)
r_metadata.zincrby(f'{decoder_name}_date:{date}', 1, self.id)
self.update_daterange(date)
# First Hash for this decoder this day
# Correlations
self.add_correlation('item', '', obj_id)
# domain
if is_crawled(obj_id):
domain = get_item_domain(obj_id)
self.add_correlation('domain', '', domain)
# Filter duplicates ######################################################################
# Filter on item + hash for this day
# filter Obj Duplicate
# first time we see this day
# iterate on all decoder
######################################################################
# first time we see this hash today
# mimetype # # # # # # # #
r_metadata.zincrby(f'decoded:mimetype:{date}', 1, mimetype)
# create hash metadata
r_metadata.sadd(f'decoded:mimetypes', mimetype)
# # TODO: DUPLICATES + check fields
def add(self, decoder_name, date, obj_id, mimetype):
self.update_daterange(date)
r_metadata.incrby(f'{decoder_type}_decoded:{date}', 1)
r_metadata.zincrby(f'{decoder_type}_date:{date}', 1, self.id)
r_metadata.hincrby(f'metadata_hash:{self.id}', f'{decoder_type}_decoder', 1)
r_metadata.zincrby(f'{decoder_type}_type:{mimetype}', 1, date) # # TODO: # DUP1
################################################################ # TODO: REMOVE ?????????????????????????????????
r_metadata.zincrby(f'{decoder_type}_hash:{self.id}', 1, obj_id) # number of b64 on this item
# first time we see this hash encoding on this item
if not r_metadata.zscore(f'{decoder_type}_hash:{self.id}', obj_id):
# create hash metadata
r_metadata.sadd(f'hash_{decoder_type}_all_type', mimetype)
# first time we see this hash encoding today
if not r_metadata.zscore(f'{decoder_type}_date:{date}', self.id):
r_metadata.zincrby(f'{decoder_type}_type:{mimetype}', 1, date) # # TODO: # DUP1
# Correlations
self.add_correlation('item', '', obj_id)
# domain
if is_crawled(obj_id):
domain = get_item_domain(obj_id)
self.add_correlation('domain', '', domain)
# NB of MIMETYPE / DAY -> ALL HASH OR UNIQ HASH ??????
# algo + mimetype -> Decodeds
# -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}')
# # TODO: ADD items
def create(self, content, date, mimetype=None):
@ -324,17 +240,14 @@ class Decoded(AbstractObject):
#######################################################################################
#######################################################################################
#######################################################################################
#######################################################################################
def is_vt_enabled(self):
return VT_ENABLED
def set_vt_report(self, report):
r_metadata.hset(f'metadata_hash:{self.id}', 'vt_report', report)
r_objects.hset(f'meta:{self.type}:{self.id}', 'vt_report', report)
def set_meta_vt(self, link, report):
r_metadata.hset(f'metadata_hash:{self.id}', 'vt_link', link)
r_objects.hset(f'meta:{self.type}:{self.id}', 'vt_link', link)
self.set_vt_report(report)
def refresh_vt_report(self):
@ -371,17 +284,68 @@ class Decoded(AbstractObject):
link = json_response['permalink'].split('analysis')[0] + 'analysis/'
self.set_meta_vt(link, 'Please Refresh')
############################################################################
############################################################################
############################################################################
def get_decoders_names():
def is_vt_enabled():
return VT_ENABLED
def get_all_decodeds():
return r_objects.smembers(f'decoded:all')
def get_algos():
return ['base64', 'binary', 'hexadecimal']
def get_all_mimetypes():
return r_metadata.smembers(f'decoded:mimetypes')
return r_objects.smembers('decoded:mimetypes')
def get_nb_decodeds_by_date(date):
return r_objects.zcard(f'decoded:date:{date}')
def get_decodeds_by_date(date):
return r_objects.zrange(f'decoded:date:{date}', 0, -1)
def get_algo_decodeds_by_date(date, algo):
return r_objects.smembers(f'decoded:algo:{algo}:{date}')
def get_mimetype_decodeds_by_date(date, mimetype):
return r_objects.smembers(f'decoded:mime:{mimetype}:{date}')
def get_algo_mimetype_decodeds_by_date(date, algo, mimetype):
return r_objects.sinter(f'decoded:algo:{algo}:{date}', f'decoded:mime:{mimetype}:{date}')
def get_decodeds_by_daterange(date_from, date_to, algo=None, mimetype=None):
decodeds = set()
if not algo and not mimetype:
for date in Date.substract_date(date_from, date_to):
decodeds = decodeds | set(get_decodeds_by_date(date))
elif algo and not mimetype:
for date in Date.substract_date(date_from, date_to):
decodeds = decodeds | get_algo_decodeds_by_date(date, algo)
elif mimetype and not algo:
for date in Date.substract_date(date_from, date_to):
decodeds = decodeds | get_mimetype_decodeds_by_date(date, mimetype)
elif algo and mimetype:
for date in Date.substract_date(date_from, date_to):
decodeds = decodeds | get_algo_mimetype_decodeds_by_date(date, algo, mimetype)
return decodeds
def sanitise_algo(algo):
if algo in get_algos():
return algo
else:
return None
def sanitise_mimetype(mimetype):
if mimetype:
if r_objects.sismember('decoded:mimetypes', mimetype):
return mimetype
else:
return None
############################################################################
def sanityze_decoder_names(decoder_name):
if decoder_name not in Decodeds.get_decoders_names():
if decoder_name not in Decodeds.get_algos():
return None
else:
return decoder_name
@ -389,11 +353,12 @@ def sanityze_decoder_names(decoder_name):
def sanityze_mimetype(mimetype):
if mimetype == 'All types':
return None
elif not r_metadata.sismember('hash_all_type', mimetype):
elif not r_objects.sismember(f'decoded:mimetypes', mimetype):
return None
else:
return mimetype
# TODO
def pie_chart_mimetype_json(date_from, date_to, mimetype, decoder_name):
if mimetype:
all_mimetypes = [mimetype]
@ -404,33 +369,121 @@ def pie_chart_mimetype_json(date_from, date_to, mimetype, decoder_name):
for mimet in all_mimetypes:
pass
# TODO
def pie_chart_decoder_json(date_from, date_to, mimetype):
all_decoder = get_decoders_names()
all_algos = get_algos()
date_range = Date.substract_date(date_from, date_to)
if not date_range:
date_range.append(Date.get_today_date_str())
nb_decoder = {}
nb_algos = {}
for date in date_range:
for decoder_name in all_decoder:
if not mimetype:
nb = r_metadata.get(f'{decoder_name}_decoded:{date}')
if nb is None:
nb = 0
else:
nb = int(nb)
for algo_name in all_algos:
# if not mimetype:
nb = r_objects.zscore(f'decoded:algos:{date}', algo_name)
# TODO mimetype necoding per day
# else:
# nb = r_metadata.zscore(f'{algo_name}_type:{mimetype}', date)
if nb is None:
nb = 0
else:
nb = r_metadata.zscore(f'{decoder_name}_type:{mimetype}', date)
nb_decoder[decoder_name] = nb_decoder.get(decoder_name, 0) + nb
nb = int(nb)
nb_algos[algo_name] = nb_algos.get(algo_name, 0) + nb
pie_chart = []
for decoder_name in all_decoder:
pie_chart.append({'name': decoder_name, 'value': nb_decoder[decoder_name]})
for algo_name in all_algos:
pie_chart.append({'name': algo_name, 'value': nb_algos[algo_name]})
return pie_chart
# TODO FILTER BY ALGO
def pie_chart_mimetype_json(date_from, date_to, algo):
date_range = Date.substract_date(date_from, date_to)
if not date_range:
date_range.append(Date.get_today_date_str())
mimetypes = {}
if len(date_range) == 1:
mimes = r_objects.zrange(f'decoded:mime:{date_range[0]}', 0, -1, withscores=True)
for t_mime in mimes:
mime, nb = t_mime
mimetypes[mime] = int(nb)
else:
mimetypes = {}
for date in date_range:
mimes = r_objects.zrange(f'decoded:mime:{date}', 0, -1, withscores=True)
for t_mime in mimes:
mime, nb = t_mime
mimetypes[mime] = mimetypes.get(mime, 0) + int(nb)
top5_mimes = sorted(mimetypes, key=mimetypes.get, reverse=True)[:5]
pie_chart = []
for mime in top5_mimes:
pie_chart.append({'name': mime, 'value': mimetypes[mime]})
return pie_chart
def barchart_range_json(date_from, date_to, mimetype=None):
date_range = Date.substract_date(date_from, date_to)
if not date_range:
date_range.append(Date.get_today_date_str())
barchart = []
if mimetype:
for date in date_range:
range_day = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'}
nb_day = r_objects.scard(f'decoded:mime:{mimetype}:{date}')
range_day[mimetype] = nb_day
barchart.append(range_day)
else:
# algo by mimetype, date = mimetype
if len(date_range) == 1:
mimes = r_objects.zrange(f'decoded:mime:{date_range[0]}', 0, -1, withscores=True)
# TODO
# UNION
# f'decoded:algo:{algo_name}:{date}'
# f'decoded:mime:{mimetype}:{date}'
for t_mime in mimes:
mime, nb = t_mime
range_day = {'date': mime, 'mimetype': nb}
barchart.append(range_day)
# mimetypes by date
else:
mimetypes = set()
for date in date_range:
range_day = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'}
mimes = r_objects.zrange(f'decoded:mime:{date}', 0, -1, withscores=True)
for t_mime in mimes:
mime, nb = t_mime
mimetypes.add(mime)
range_day[mime] = int(nb)
barchart.append(range_day)
if not mimetypes:
mimetypes.add('No Data')
for row in barchart:
for mime in mimetypes:
if mime not in row:
row[mime] = 0
return barchart
def graphline_json(decoded_id):
decoded = Decoded(decoded_id)
graphline = []
if decoded.exists():
nb_day = 30
for date in Date.get_previous_date_list(nb_day):
graphline.append({'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', 'value': decoded.get_nb_seen_by_date(date)})
return graphline
def api_pie_chart_decoder_json(date_from, date_to, mimetype):
mimetype = sanityze_mimetype(mimetype)
date = Date.sanitise_date_range(date_from, date_to)
return pie_chart_decoder_json(date['date_from'], date['date_to'], mimetype)
def api_pie_chart_mimetype_json(date_from, date_to, algo):
algo = sanitise_algo(algo)
date = Date.sanitise_date_range(date_from, date_to)
return pie_chart_mimetype_json(date['date_from'], date['date_to'], algo)
def api_barchart_range_json(date_from, date_to, mimetype):
date = Date.sanitise_date_range(date_from, date_to)
if mimetype:
mimetype = sanityze_mimetype(mimetype)
return barchart_range_json(date['date_from'], date['date_to'], mimetype=mimetype)
def _delete_old_json_descriptor():
decodeds = []
hash_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR)
@ -441,15 +494,17 @@ def _delete_old_json_descriptor():
os.remove(decoded_path)
return decodeds
def get_all_decodeds():
# TODO
def get_all_decodeds_files():
decodeds = []
hash_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR)
if not hash_dir.endswith("/"):
hash_dir = f"{hash_dir}/"
for root, dirs, files in os.walk(hash_dir):
for file in files:
decoded_path = f'{root}{file}'
# decoded_path = f'{root}{file}'
decodeds.append(file)
return decodeds
#if __name__ == '__main__':
# if __name__ == '__main__':

View file

@ -65,7 +65,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
return last_seen
def get_nb_seen(self):
return r_object.hget(f'meta:{self.type}:{self.id}', 'nb')
return self.get_nb_correlation('item')
def get_nb_seen_by_date(self, date):
nb = r_object.zscore(f'{self.type}:date:{date}', self.id)
@ -108,20 +108,23 @@ class AbstractDaterangeObject(AbstractObject, ABC):
sparkline.append(self.get_nb_seen_by_date(date))
return sparkline
def _add_create(self):
r_object.sadd(f'{self.type}:all', self.id)
# TODO don't increase nb if same hash in item with different encoding
# if hash already in item
def _add(self, date, item_id):
if not self.exists():
self._add_create(date)
self.set_first_seen(date)
self.set_last_seen(date)
r_object.sadd(f'{self.type}:all', self.id)
else:
self.update_daterange(date)
update_obj_date(date, self.type)
# NB Object seen by day
print(f'{self.type}:date:{date}', 1, self.id)
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
# NB Object seen
r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1)
if not self.is_correlated('item', '', item_id): # if decoded not already in object
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
# Correlations
self.add_correlation('item', '', item_id)

View file

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib import Tag
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
@ -219,6 +219,9 @@ class AbstractObject(ABC):
"""
return get_correlations(self.type, self.subtype, self.id)
def get_nb_correlation(self, correl_type):
return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type)
def get_nb_correlations(self, filter_types=[]):
return get_nb_correlations(self.type, self.subtype, self.id, filter_types=filter_types)

View file

@ -97,6 +97,21 @@ def get_object_svg(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.get_svg_icon()
## TAGS ##
def get_obj_tags(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.get_tags()
def add_obj_tag(obj_type, subtype, id, tag):
obj = get_object(obj_type, subtype, id)
obj.add_tag(tag)
def add_obj_tags(obj_type, subtype, id, tags):
obj = get_object(obj_type, subtype, id)
for tag in tags:
obj.add_tag(tag)
# -TAGS- #
def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
obj = get_object(obj_type, subtype, id)
meta = obj.get_meta(options=options)
@ -117,6 +132,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
meta['icon'] = obj.get_svg_icon()
if subtype or obj_type == 'cve':
meta['sparkline'] = obj.get_sparkline()
meta['cve_search'] = obj.get_cve_search()
if subtype == 'bitcoin' and related_btc:
meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id)
if obj.get_type() == 'decoded':

View file

@ -9,15 +9,10 @@
##################################
# Import External packages
##################################
import time
import os
import base64
from hashlib import sha1
import magic
import json
import datetime
import re
import signal
import sys
sys.path.append(os.environ['AIL_BIN'])
@ -27,49 +22,27 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib.objects import Decodeds
from lib.objects.Decodeds import Decoded
config_loader = ConfigLoader()
serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
hex_max_execution_time = config_loader.get_config_int("Hex", "max_execution_time")
binary_max_execution_time = config_loader.get_config_int("Binary", "max_execution_time")
base64_max_execution_time = config_loader.get_config_int("Base64", "max_execution_time")
config_loader = None
#####################################################
#####################################################
# # TODO: use regex_helper
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
# # TODO: # FIXME: Remove signal -> replace with regex_helper
signal.signal(signal.SIGALRM, timeout_handler)
#####################################################
####################################################
class Decoder(AbstractModule):
"""
Decoder module for AIL framework
"""
# TODO to lambda expr
def hex_decoder(self, hexStr):
#hexStr = ''.join( hex_string.split(" ") )
# hexStr = ''.join( hex_string.split(" ") )
return bytes(bytearray([int(hexStr[i:i+2], 16) for i in range(0, len(hexStr), 2)]))
# TODO to lambda expr
def binary_decoder(self, binary_string):
return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)]))
# TODO to lambda expr
def base64_decoder(self, base64_string):
return base64.b64decode(base64_string)
@ -86,18 +59,20 @@ class Decoder(AbstractModule):
cmp_regex_base64 = re.compile(regex_base64)
# map decoder function
self.decoder_function = {'binary': self.binary_decoder, 'hexadecimal': self.hex_decoder, 'base64': self.base64_decoder}
self.decoder_function = {'binary': self.binary_decoder,
'hexadecimal': self.hex_decoder,
'base64': self.base64_decoder}
# list all decoder with regex,
decoder_binary = {'name': 'binary', 'regex': cmp_regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
decoder_hexadecimal = {'name': 'hexadecimal', 'regex': cmp_regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
decoder_base64 = {'name': 'base64', 'regex': cmp_regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}
decoder_binary = {'name': 'binary', 'regex': cmp_regex_binary,
'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
decoder_hexadecimal = {'name': 'hexadecimal', 'regex': cmp_regex_hex,
'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
decoder_base64 = {'name': 'base64', 'regex': cmp_regex_base64,
'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}
self.decoder_order = [decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64]
for decoder in self.decoder_order:
serv_metadata.sadd('all_decoder', decoder['name'])
# Waiting time in seconds between to message processed
self.pending_seconds = 1
@ -110,63 +85,47 @@ class Decoder(AbstractModule):
content = item.get_content()
date = item.get_date()
for decoder in self.decoder_order: # add threshold and size limit
# max execution time on regex
signal.alarm(decoder['max_execution_time'])
for decoder in self.decoder_order:
find = False
dname = decoder['name']
try:
encoded_list = decoder['regex'].findall(content)
except TimeoutException:
encoded_list = []
self.process.incr_module_timeout_statistic() # add encoder type
self.redis_logger.debug(f"{item.id} processing timeout")
continue
else:
signal.alarm(0)
if len(encoded_list) > 0:
content = self.decode_string(content, item.id, date, encoded_list, decoder['name'], decoder['encoded_min_size'])
def decode_string(self, content, item_id, date, encoded_list, decoder_name, encoded_min_size):
find = False
for encoded in encoded_list:
if len(encoded) >= encoded_min_size:
decoded_file = self.decoder_function[decoder_name](encoded)
find = True
sha1_string = sha1(decoded_file).hexdigest()
decoded = Decoded(sha1_string)
mimetype = decoded.guess_mimetype(decoded_file)
if not mimetype:
print(item_id)
print(sha1_string)
raise Exception(f'Invalid mimetype: {sha1_string} {item_id}')
decoded.create(content, date)
decoded.add(decoder_name, date, item_id, mimetype)
save_item_relationship(sha1_string, item_id) ################################
# remove encoded from item content
encodeds = self.regex_findall(decoder['regex'], item.id, content)
# PERF remove encoded from item content
for encoded in encodeds:
content = content.replace(encoded, '', 1)
encodeds = set(encodeds)
self.redis_logger.debug(f'{item_id} : {decoder_name} - {mimetype}')
print(f'{item_id} : {decoder_name} - {mimetype}')
if find:
self.redis_logger.info(f'{decoder_name} decoded')
print(f'{decoder_name} decoded')
for encoded in encodeds:
find = False
if len(encoded) >= decoder['encoded_min_size']:
decoded_file = self.decoder_function[dname](encoded)
# Send to Tags
msg = f'infoleak:automatic-detection="{decoder_name}";{item_id}'
self.send_message_to_queue(msg, 'Tags')
sha1_string = sha1(decoded_file).hexdigest()
decoded = Decoded(sha1_string)
# perf: remove encoded from item content
return content
if not decoded.exists():
mimetype = decoded.guess_mimetype(decoded_file)
if not mimetype:
print(sha1_string, item.id)
raise Exception(f'Invalid mimetype: {decoded.id} {item.id}')
decoded.save_file(decoded_file, mimetype)
else:
mimetype = decoded.get_mimetype()
decoded.add(dname, date, item.id, mimetype=mimetype)
# DEBUG
self.redis_logger.debug(f'{item.id} : {dname} - {decoded.id} - {mimetype}')
print(f'{item.id} : {dname} - {decoded.id} - {mimetype}')
if find:
self.redis_logger.info(f'{item.id} - {dname}')
print(f'{item.id} - {dname}')
# Send to Tags
msg = f'infoleak:automatic-detection="{dname}";{item.id}'
self.send_message_to_queue(msg, 'Tags')
if __name__ == '__main__':
# # TODO: TEST ME
module = Decoder()
module.run()

View file

@ -87,7 +87,7 @@ class AbstractModule(ABC):
def regex_finditer(self, regex, obj_id, content):
return regex_helper.regex_finditer(self.r_cache_key, regex, obj_id, content, max_time=self.max_execution_time)
def regex_findall(self, regex, id, content):
def regex_findall(self, regex, id, content, r_set=False):
"""
regex findall helper (force timeout)
:param regex: compiled regex
@ -96,7 +96,7 @@ class AbstractModule(ABC):
ex: send_to_queue(item_id, 'Global')
"""
return regex_helper.regex_findall(self.module_name, self.r_cache_key, regex, id, content, max_time=self.max_execution_time)
return regex_helper.regex_findall(self.module_name, self.r_cache_key, regex, id, content, max_time=self.max_execution_time, r_set=r_set)
def run(self):
"""

View file

@ -183,3 +183,42 @@ def sanitise_date_range(date_from, date_to, separator='', date_type='str'):
date_from = date_to
date_to = res
return {"date_from": date_from, "date_to": date_to}
def sanitise_daterange(date_from, date_to, separator='', date_type='str'):
'''
Check/Return a correct date_form and date_to
'''
if not date_from and date_to:
date_from = date_to
elif not date_to and date_from:
date_to = date_from
elif not date_to and not date_from:
date = datetime.date.today().strftime("%Y%m%d")
return date, date
if date_type == 'str':
# remove separators
if len(date_from) == 10:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
if len(date_to) == 10:
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
if not validate_str_date(date_from, separator=separator):
date_from = datetime.date.today().strftime("%Y%m%d")
if not validate_str_date(date_to, separator=separator):
date_to = datetime.date.today().strftime("%Y%m%d")
else: # datetime
if isinstance(date_from, datetime.datetime):
date_from = date_from.strftime("%Y%m%d")
else:
date_from = datetime.date.today().strftime("%Y%m%d")
if isinstance(date_to, datetime.datetime):
date_to = date_to.strftime("%Y%m%d")
else:
date_to = datetime.date.today().strftime("%Y%m%d")
if int(date_from) > int(date_to):
res = date_from
date_from = date_to
date_to = res
return date_from, date_to

View file

@ -83,5 +83,15 @@ def objects_cve_search():
else:
return redirect(cve.get_link(flask_context=True))
@objects_cve.route("/objects/cve/graphline/json", methods=['GET'])
@login_required
@login_read_only
def objects_cve_graphline_json():
cve_id = request.args.get('id')
cve = Cves.Cve(cve_id)
if not cve.exists():
abort(404)
return jsonify(Cves.get_cve_graphline(cve_id))
# ============= ROUTES ==============

View file

@ -20,10 +20,12 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.objects import Decodeds
from packages import Date
# ============ BLUEPRINT ============
objects_decoded = Blueprint('objects_decoded', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/decoded'))
objects_decoded = Blueprint('objects_decoded', __name__,
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/decoded'))
# ============ VARIABLES ============
@ -33,9 +35,60 @@ objects_decoded = Blueprint('objects_decoded', __name__, template_folder=os.path
# ============= ROUTES ==============
# # TODO: # FIXME: CHECK IF OBJ EXIST
@objects_decoded.route("/object/decodeds", methods=['GET', 'POST'])
@login_required
@login_read_only
def decodeds_dashboard():
if request.method == 'POST':
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
mimetype = request.form.get('mimetype')
algo = request.form.get('algo')
show_decoded = request.form.get('show_decoded')
return redirect(
url_for('objects_decoded.decodeds_dashboard', date_from=date_from, date_to=date_to, mimetype=mimetype,
algo=algo, show=show_decoded))
else:
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
mimetype = request.args.get('mimetype')
algo = request.args.get('algo')
show_decoded = request.args.get('show')
if show_decoded:
show_decoded = True
@objects_decoded.route("/object/decoded/download") #completely shows the paste in a new tab
if mimetype == 'All types':
mimetype = None
if algo == 'All encoding':
algo = None
algo = Decodeds.sanitise_algo(algo)
mimetype = Decodeds.sanitise_mimetype(mimetype)
date_from, date_to = Date.sanitise_daterange(date_from, date_to)
metas = []
if show_decoded:
decodeds = Decodeds.get_decodeds_by_daterange(date_from, date_to, algo=algo, mimetype=mimetype)
metas = []
for decoded_id in decodeds:
decoded = Decodeds.Decoded(decoded_id)
metas.append(decoded.get_meta(options={'sparkline', 'mimetype', 'icon', 'size', 'vt'}))
# TODO GET PIE CHARTS
return render_template("decoded/decodeds_dashboard.html", metas=metas, vt_enabled=Decodeds.is_vt_enabled(),
date_from=date_from, date_to=date_to, algo=algo, mimetype=mimetype,
algos=Decodeds.get_algos(), show_decoded=show_decoded,
mimetypes=Decodeds.get_all_mimetypes())
@objects_decoded.route("/object/decodeds/search", methods=['POST'])
@login_required
@login_read_only
def decodeds_search():
decoded_id = request.form.get('object_id')
print(decoded_id)
return redirect(url_for('correlation.show_correlation', type='decoded', id=decoded_id))
@objects_decoded.route("/object/decoded/download")
@login_required
@login_read_only
def decoded_download():
@ -51,7 +104,8 @@ def decoded_download():
else:
abort(404)
@objects_decoded.route("/object/decoded/send_to_vt") #completely shows the paste in a new tab
@objects_decoded.route("/object/decoded/send_to_vt")
@login_required
@login_read_only
def send_to_vt():
@ -66,7 +120,8 @@ def send_to_vt():
else:
abort(404)
@objects_decoded.route("/object/decoded/refresh_vt_report") #completely shows the paste in a new tab
@objects_decoded.route("/object/decoded/refresh_vt_report")
@login_required
@login_read_only
def refresh_vt_report():
@ -81,18 +136,42 @@ def refresh_vt_report():
else:
abort(404)
@objects_decoded.route("/object/decoded/decoder_pie_chart_json", methods=['GET'])
# TODO
@objects_decoded.route("/object/decoded/algo_pie_chart/json", methods=['GET'])
@login_required
@login_read_only
def decoder_pie_chart_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
mimetype = request.args.get('type')
mimetype = request.args.get('mimetype')
return jsonify(Decodeds.api_pie_chart_decoder_json(date_from, date_to, mimetype))
# TODO
@objects_decoded.route("/object/decoded/mimetype_pie_chart/json", methods=['GET'])
@login_required
@login_read_only
def mimetype_pie_chart_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
algo = request.args.get('algo')
return jsonify(Decodeds.api_pie_chart_mimetype_json(date_from, date_to, algo))
@objects_decoded.route("/object/decoded/barchart/json", methods=['GET'])
@login_required
@login_read_only
def barchart_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
mimetype = request.args.get('mimetype')
return jsonify(Decodeds.api_barchart_range_json(date_from, date_to , mimetype))
#####################################################3
@objects_decoded.route("/object/decoded/graphline/json", methods=['GET'])
@login_required
@login_read_only
def graphline_json():
decoded_id = request.args.get('id')
decoded = Decodeds.Decoded(decoded_id)
if not decoded:
abort(404)
return jsonify(Decodeds.graphline_json(decoded_id))

View file

@ -59,46 +59,6 @@ def substract_date(date_from, date_to):
l_date.append( date.strftime('%Y%m%d') )
return l_date
def list_sparkline_values(date_range_sparkline, hash):
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date_day, hash)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_file_icon(estimated_type):
file_type = estimated_type.split('/')[0]
# set file icon
if file_type == 'application':
file_icon = 'fa-file '
elif file_type == 'audio':
file_icon = 'fa-file-audio '
elif file_type == 'image':
file_icon = 'fa-file-image'
elif file_type == 'text':
file_icon = 'fa-file-alt'
else:
file_icon = 'fa-sticky-note'
return file_icon
def get_file_icon_text(estimated_type):
file_type = estimated_type.split('/')[0]
# set file icon
if file_type == 'application':
file_icon_text = '\uf15b'
elif file_type == 'audio':
file_icon_text = '\uf1c7'
elif file_type == 'image':
file_icon_text = '\uf1c5'
elif file_type == 'text':
file_icon_text = '\uf15c'
else:
file_icon_text = '\uf249'
return file_icon_text
def get_icon(correlation_type, type_id):
icon_text = 'fas fa-sticky-note'
@ -182,16 +142,6 @@ def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_i
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_all_keys_id_from_item(correlation_type, item_path):
all_keys_id_dump = set()
if item_path is not None:
for type_id in get_all_types_id(correlation_type):
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_type, type_id, item_path))
for key_id in res:
all_keys_id_dump.add( (key_id, type_id) )
return all_keys_id_dump
def get_correlation_type_search_endpoint(correlation_type):
if correlation_type == 'pgpdump':
endpoint = 'hashDecoded.all_pgpdump_search'
@ -379,345 +329,7 @@ def correlation_type_range_type_json(correlation_type, date_from, date_to):
return jsonify(range_type)
# ============= ROUTES ==============
@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
@login_required
@login_read_only
def all_hash_search():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
type = request.form.get('type')
encoding = request.form.get('encoding')
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
@hashDecoded.route("/hashDecoded/", methods=['GET'])
@login_required
@login_read_only
def hashDecoded_page():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
type = request.args.get('type')
encoding = request.args.get('encoding')
show_decoded_files = request.args.get('show_decoded_files')
if type == 'All types':
type = None
if encoding == 'All encoding':
encoding = None
#date_from = '20180628' or date_from = '2018-06-28'
#date_to = '20180628' or date_to = '2018-06-28'
# verify file type input
if type is not None:
#retrieve + char
type = type.replace(' ', '+')
if type not in r_serv_metadata.smembers('hash_all_type'):
type = None
all_encoding = r_serv_metadata.smembers('all_decoder')
# verify encoding input
if encoding is not None:
if encoding not in all_encoding:
encoding = None
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
# display day type bar chart
if len(date_range) == 1 and type is None:
daily_type_chart = True
daily_date = date_range[0]
else:
daily_type_chart = False
daily_date = None
l_64 = set()
if show_decoded_files:
show_decoded_files = True
for date in date_range:
if encoding is None:
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
else:
l_hash = r_serv_metadata.zrange(encoding+'_date:' +date, 0, -1)
if l_hash:
for hash in l_hash:
l_64.add(hash)
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
b64_metadata = []
l_64 = list(l_64)
for hash in l_64:
# select requested base 64 type
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
if type is not None:
if estimated_type is not None:
if estimated_type != type:
continue
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
if hash is not None and first_seen is not None and \
last_seen is not None and \
nb_seen_in_paste is not None and \
size is not None:
file_icon = get_file_icon(estimated_type)
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
b64_vt = True
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
else:
b64_vt = False
b64_vt_link = ''
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
# hash never refreshed
if b64_vt_report is None:
b64_vt_report = ''
sparklines_value = list_sparkline_values(date_range_sparkline, hash)
b64_metadata.append( (file_icon, estimated_type, hash, nb_seen_in_paste, size, first_seen, last_seen, b64_vt, b64_vt_link, b64_vt_report, sparklines_value) )
l_type = sorted(r_serv_metadata.smembers('hash_all_type'))
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files)
@hashDecoded.route('/hashDecoded/hash_by_type_json')
@login_required
@login_read_only
def hash_by_type_json():
type = request.args.get('type')
#retrieve + char
type = type.replace(' ', '+')
num_day_type = 30
date_range = get_date_range(num_day_type)
#verify input
if type in r_serv_metadata.smembers('hash_all_type'):
type_value = []
all_decoder = r_serv_metadata.smembers('all_decoder')
range_decoder = []
for date in date_range:
day_decoder = {}
day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for decoder in all_decoder:
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_decoder is None:
num_day_decoder = 0
day_decoder[decoder]= num_day_decoder
range_decoder.append(day_decoder)
return jsonify(range_decoder)
else:
return jsonify()
@hashDecoded.route('/hashDecoded/top5_type_json')
@login_required
@login_read_only
def top5_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
typ = request.args.get('type')
decoder = request.args.get('encoding')
if decoder == 'All encoding' or decoder is None:
all_decoder = r_serv_metadata.smembers('all_decoder')
else:
if not r_serv_metadata.sismember('all_decoder', decoder):
return jsonify({'Error': 'This decoder do not exist'})
else:
all_decoder = [decoder]
if typ == 'All types' or typ is None or typ=='None':
all_type = r_serv_metadata.smembers('hash_all_type')
else:
typ = typ.replace(' ', '+')
if not r_serv_metadata.sismember('hash_all_type', typ):
return jsonify({'Error': 'This type do not exist'})
else:
all_type = [typ]
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
# TODO replace with ZUNIONSTORE
nb_types_decoded = {}
for date in date_range:
for typ in all_type:
for decoder in all_decoder:
nb_decoded = r_serv_metadata.zscore(f'{decoder}_type:{typ}', date) # daily_type key:date mimetype 3
if nb_decoded is not None:
if typ in nb_types_decoded:
nb_types_decoded[typ] = nb_types_decoded[typ] + int(nb_decoded)
else:
nb_types_decoded[typ] = int(nb_decoded)
to_json = []
top5_types = sorted(nb_types_decoded, key=nb_types_decoded.get, reverse=True)[:5]
for typ in top5_types:
to_json.append({'name': typ, 'value': nb_types_decoded[typ]})
return jsonify(to_json)
@hashDecoded.route('/hashDecoded/daily_type_json')
@login_required
@login_read_only
def daily_type_json():
date = request.args.get('date')
daily_type = set()
l_b64 = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
for hash in l_b64:
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
if estimated_type is not None:
daily_type.add(estimated_type)
type_value = []
for day_type in daily_type:
num_day_type = r_serv_metadata.zscore('hash_type:'+day_type, date)
type_value.append({ 'date' : day_type, 'value' : int( num_day_type )})
return jsonify(type_value)
@hashDecoded.route('/hashDecoded/range_type_json')
@login_required
@login_read_only
def range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
date_range = []
if date_from is not None and date_to is not None:
#change format
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
all_type = set()
for date in date_range:
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
if l_hash:
for hash in l_hash:
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
if estimated_type:
all_type.add(estimated_type)
range_type = []
list_decoder = r_serv_metadata.smembers('all_decoder')
for date in date_range:
if len(date_range) == 1:
if date==date_from and date==date_to:
for type in all_type:
day_type = {}
day_type['date']= type
for decoder in list_decoder:
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_decoder is None:
num_day_decoder = 0
day_type[decoder]= num_day_decoder
range_type.append(day_type)
else:
range_type = ''
else:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type in all_type:
num_day_type = 0
for decoder in list_decoder:
num_day_type_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_type_decoder is not None:
num_day_type += num_day_type_decoder
day_type[type]= num_day_type
range_type.append(day_type)
return jsonify(range_type)
@hashDecoded.route('/hashDecoded/hash_graph_line_json')
@login_required
@login_read_only
def hash_graph_line_json():
hash = request.args.get('hash')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
if date_from is None or date_to is None:
nb_days_seen_in_pastes = 30
else:
# # TODO: # FIXME:
nb_days_seen_in_pastes = 30
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
# verify input
if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None:
json_seen_in_paste = []
for date in date_range_seen_in_pastes:
nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash)
if nb_seen_this_day is None:
nb_seen_this_day = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
return jsonify(json_seen_in_paste)
else:
return jsonify()
############################ PGPDump ############################

View file

@ -594,8 +594,7 @@ function barchart_type(url, id) {
.attr("transform", "rotate(-20)" );
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d });
.attr("class", "bar");
{% endif %}
svg.append("g")
@ -617,12 +616,6 @@ function barchart_type(url, id) {
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
{% if type %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
{% endif %}
{% if daily_type_chart %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
{% endif %}
data.forEach(function(d) {

View file

@ -1 +1 @@
<li id='page-hashDecoded'><a href="{{ url_for('hashDecoded.hashDecoded_page') }}"><i class="fa fa-files-o"></i> hashesDecoded </a></li>
<li id='page-hashDecoded'><a href="{{ url_for('objects_decoded.decodeds_dashboard') }}"><i class="fa fa-files-o"></i> hashesDecoded </a></li>

View file

@ -270,9 +270,9 @@ $(document).ready(function(){
{% elif dict_object["object_type"] == "cryptocurrency" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.cryptocurrency_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "decoded" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{dict_object["correlation_id"]}}");
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_decoded.graphline_json') }}?id={{dict_object["correlation_id"]}}");
{% elif dict_object["object_type"] == "cve" %}
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{dict_object["correlation_id"]}}");
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_cve.objects_cve_graphline_json') }}?id={{dict_object["correlation_id"]}}");
{% endif %}
all_graph.onResize();
});

View file

@ -0,0 +1,703 @@
<!DOCTYPE html>
<html>
<head>
<title>Decoded - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
.input-group .form-control {
position: unset;
}
.line {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.bar {
fill: steelblue;
}
.bar:hover{
fill: brown;
cursor: pointer;
}
.bar_stack:hover{
cursor: pointer;
}
.pie_path:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="row">
<div class="col-xl-10">
<div class="mt-1" id="barchart_type">
</div>
<div class="card border-secondary my-2">
<div class="card-body text-dark">
<h5 class="card-title">Search Decoded by name:</h5>
<form action="{{ url_for('objects_decoded.decodeds_search') }}" id="search_subtype_onj" method='post'>
<div class="input-group mb-1">
<input type="text" class="form-control col-8" name="object_id" value="" placeholder="Decoded ID" required>
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
</div>
</form>
</div>
</div>
</div>
<div class="col-xl-2">
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
<div class="card-body text-center py-2">
<h6 class="card-title" style="color:#286090;">Select a date range :</h6>
<form action="{{ url_for('objects_decoded.decodeds_dashboard') }}" id="hash_selector_form" method='post'>
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{% if date_from %}{{ date_from[0:4] }}-{{ date_from[4:6] }}-{{ date_from[6:8] }}{% endif %}" name="date_from" autocomplete="off">
</div>
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{% if date_to %}{{ date_to[0:4] }}-{{ date_to[4:6] }}-{{ date_to[6:8] }}{% endif %}" name="date_to" autocomplete="off">
</div>
<div class="mt-1" style="font-size: 14px;color:#286090;">Encoding :</div>
<select class="custom-select" name="algo">
<option>All encoding</option>
{% for alg in algos %}
{% if alg|string() == algo|string() %}
<option selected>{{ alg }}</option>
{% else %}
<option>{{ alg }}</option>
{% endif %}
{% endfor %}
</select>
<div class="mt-1" style="font-size: 14px;color:#286090;">File Type :</div>
<select class="custom-select" name="mimetype">
<option>All types</option>
{% for typ in mimetypes %}
{% if mimetype|string() == typ|string() %}
<option selected>{{ typ }}</option>
{% else %}
<option>{{ typ }}</option>
{% endif %}
{% endfor %}
</select>
<div class="form-check my-1">
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded" value="True" {% if show_decoded %}checked{% endif %}>
<label class="form-check-label" for="checkbox-input-show">
<span style="color:#286090; font-size: 14px;">
Show decoded files <i class="fas fa-file"></i>
</span>
</label>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fas fa-copy"></i> Search
</button>
</form>
</div>
</div>
<div id="pie_chart_encoded">
</div>
<div id="pie_chart_top5_types">
</div>
</div>
</div>
{% if metas|length != 0 %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }} Decoded files: </h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }} Decoded files: </h3>
{% endif %}
<table id="tableb64" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>estimated type</th>
<th>hash</th>
<th>first seen</th>
<th>last seen</th>
<th>nb item</th>
<th>size</th>
<th>Virus Total</th>
<th>Sparkline</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for meta in metas %}
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ meta['icon']['color'] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta['icon']['style'] }}" font-size="16px">{{ meta['icon']['icon'] }}</text>
</g>
</svg>&nbsp;&nbsp;{{ meta['mimetype'] }}
</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=decoded&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
<td>{{ meta['first_seen'] }}</td>
<td>{{ meta['last_seen'] }}</td>
<td>{{ meta['nb_seen'] }}</td>
<td>{{ meta['size'] }}</td>
<td>
{% if vt_enabled %}
{% if not meta['vt'] %}
<darkbutton_{{ meta['id'] }}>
<button id="submit_vt_{{ meta['id'] }}" class="btn btn-secondary" style="font-size: 14px;" onclick="sendFileToVT('{{ meta['id'] }}')">
<i class="fas fa-paper-plane"></i>&nbsp;Send this file to VT
</button>
</darkbutton_{{ meta['id'] }}>
{% else %}
<a class="btn btn-secondary" target="_blank" href="{{ meta['vt']['link'] }}" style="font-size: 14px;"><i class="fas fa-link"></i>&nbsp;VT Report</a>
{% endif %}
<button class="btn btn-outline-dark" onclick="updateVTReport('{{ meta['id'] }}')" style="font-size: 14px;">
<span id="report_vt_{{ meta['id'] }}"><i class="fas fa-sync-alt"></i>&nbsp;{{ meta['vt']['report'] }}</span>
</button>
{% else %}
Virus Total submission is disabled
{% endif %}
</td>
<td id="sparklines_{{ meta['id'] }}" style="text-align:center;"></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
{% if show_decoded %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No Decoded</h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }}, No Decodeds</h3>
{% endif %}
{% endif %}
{% endif %}
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_dashboard").addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
{% if date_from %}
$('#date-range-from').val("{{ date_from[0:4] }}-{{ date_from[4:6] }}-{{ date_from[6:8] }}");
{% endif %}
{% if date_to %}
$('#date-range-to').val("{{ date_to[0:4] }}-{{ date_to[4:6] }}-{{ date_to[6:8] }}");
{% endif %}
$('#tableb64').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
});
{% if mimetype %}
chart.stackBarChart =barchart_type_stack("{{ url_for('objects_decoded.barchart_json') }}?date_from={{date_from}}&date_to={{date_to}}&mimetype={{mimetype}}", 'id');
{% elif date_from==date_to and date_from %}
chart.stackBarChart =barchart_type_stack("{{ url_for('objects_decoded.barchart_json') }}?date_from={{date_from}}&date_to={{date_from}}", 'id');
{% else %}
chart.stackBarChart = barchart_type_stack("{{ url_for('objects_decoded.barchart_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
{% endif %}
draw_pie_chart("pie_chart_encoded" ,"{{ url_for('objects_decoded.decoder_pie_chart_json') }}?date_from={{date_from}}&date_to={{date_to}}&mimetype={{mimetype}}", "{{ url_for('objects_decoded.decodeds_dashboard') }}?date_from={{date_from}}&date_to={{date_to}}&mimetype={{type}}");
draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('objects_decoded.mimetype_pie_chart_json') }}?date_from={{date_from}}&date_to={{date_to}}&mimetype={{mimetype}}", "{{ url_for('objects_decoded.decodeds_dashboard') }}?date_from={{date_from}}&date_to={{date_to}}");
chart.onResize();
$(window).on("resize", function() {
chart.onResize();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
function updateVTReport(hash) {
//updateReport
$.getJSON("{{ url_for('objects_decoded.refresh_vt_report') }}?id="+hash,
function(data) {
content = '<i class="fas fa-sync-alt"></i> ' +data['report']
$( "#report_vt_"+hash ).html(content);
});
}
function sendFileToVT(hash) {
//send file to vt
$.getJSON("{{ url_for('objects_decoded.send_to_vt') }}?id="+hash,
function(data) {
var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
$('#submit_vt_'+hash).remove();
$('darkbutton_'+hash).append(content);
});
}
</script>
<script>
{% for meta in metas %}
sparkline("sparklines_{{ meta['id'] }}", {{ meta['sparkline'] }}, {});
{% endfor %}
</script>
<script>
var margin = {top: 20, right: 100, bottom: 55, left: 45},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type_stack(url, id) {
d3.json(url)
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("class", "bar")
{% if date_from|string == date_to|string and mimetype is none %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}?date_from={{date_from}}&date_to={{date_to}}&mimetype="+d })
.attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and mimetype is not none %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-18)" )
{% else %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-40)" )
{% endif %}
.style("text-anchor", "end");
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and mimetype is none %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}" +'?date_from={{date_from}}&date_to={{date_to}}&mimetype='+d.label+'&algo='+d.name; });
{% elif date_from|string == date_to|string and mimetype is not none %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}" +'?mimetype={{mimetype}}&date_from='+d.label+'&date_to='+d.label+'&algo='+d.name; });
{% else %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}" +'?mimetype='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
{% endif %}
data.forEach(function(d) {
if(d.total != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total))
{% if date_from|string == date_to|string and mimetype is none %}
.on("click", function () {window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}"+'?date_from={{date_from}}&date_to={{date_to}}&mimetype='+d.date })
{% elif date_from|string == date_to|string and mimetype is not none %}
.on("click", function () {window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}?mimetype={{mimetype}}&date_from="+d.date+'&date_to='+d.date })
{% else %}
.on("click", function () {window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}"+'?date_from='+d.date+'&date_to='+d.date })
{% endif %}
.style("text-anchor", "middle")
.text(d.total);
}
});
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: "<b><span id='tooltip-id-name-bar'></span></b>",
placement: 'top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "<span id='tooltip-id-label'></span>" +
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
});
$(this).popover('show');
$("#tooltip-id-name-bar").text(d.name);
$("#tooltip-id-label").text(d.label);
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
}
chart.onResize = function () {
var aspect = 1000 / 500, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.chart = chart;
</script>
<script>
function draw_pie_chart(id, url_json, pie_on_click_url) {
var width_pie = 200;
var height_pie = 200;
var padding_pie = 10;
var opacity_pie = .8;
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
var div_pie = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var svg_pie = d3.select("#"+id)
.append('svg')
.attr("width", '100%')
.attr("height", '100%')
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
.attr('preserveAspectRatio','xMinYMin')
var g_pie = svg_pie.append('g')
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
var arc_pie = d3.arc()
.innerRadius(0)
.outerRadius(radius_pie);
d3.json(url_json)
.then(function(data){
var pie_pie = d3.pie()
.value(function(d) { return d.value; })
.sort(null);
var path_pie = g_pie.selectAll('path')
.data(pie_pie(data))
.enter()
.append("g")
.append('path')
.attr('d', arc_pie)
.attr('fill', (d,i) => color_pie(i))
.attr('class', 'pie_path')
.on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
.style('opacity', opacity_pie)
.style('stroke', 'white');
});
function mouseovered_pie(d) {
//remove old content
$("#tooltip-id-name").remove();
$("#tooltip-id-value").remove();
// tooltip
var content;
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
div_pie.transition()
.duration(200)
.style("opacity", .9);
div_pie.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.data.name);
$("#tooltip-id-value").text(d.data.value);
}
function mouseouted_pie() {
div_pie.transition()
.duration(500)
.style("opacity", 0);
}
}
</script>
<script>
function barchart_type(url, id) {
var margin = {top: 20, right: 20, bottom: 70, left: 40};
var width = 960 - margin.left - margin.right;
var height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x)
//.tickFormat(d3.time.format("%Y-%m"));
var yAxis = d3.axisLeft(y)
.ticks(10);
/*var svg = d3.select(id).append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.attr("id", "thesvg")
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");*/
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.value = +d.value;
});
x.domain(data.map(function(d) { return d.date; }));
y.domain([0, d3.max(data, function(d) { return d.value; })]);
var label = svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", "-.55em")
{% if date_from == date_to and date_from %}
.attr("transform", "rotate(-20)" );
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
.on("click", function (d) { window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}"+'?date_from='+d+'&date_to='+d });
{% endif %}
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Value ($)");
var bar = svg.selectAll("bar")
.data(data)
.enter().append("rect")
.attr("class", "bar")
//.style("fill", "steelblue")
.attr("x", function(d) { return x(d.date); })
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
{% if mimetype %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}" +'?mimetype={{mimetype}}&date_from='+ d.date +'&date_to='+ d.date; });
{% endif %}
{% if date_from == date_to and date_from %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_decoded.decodeds_dashboard') }}" +'?mimetype='+d.date+'&date_from={{ date_from }}&date_to={{ date_from }}'; });
{% endif %}
data.forEach(function(d) {
if(d.value != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
//.text(function(d) { return d.value; });
.text(d.value)
.style("text-anchor", "middle")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.value));
}
});
});
}
</script>
</body>
</html>

View file

@ -564,7 +564,6 @@ function barchart_type(url, id) {
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d });
{% endif %}
svg.append("g")
@ -586,12 +585,6 @@ function barchart_type(url, id) {
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
{% if type %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
{% endif %}
{% if daily_type_chart %}
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
{% endif %}
data.forEach(function(d) {

View file

@ -35,7 +35,7 @@
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
<a class="nav-link" href="{{url_for('objects_decoded.decodeds_dashboard')}}" id="nav_dashboard">
<i class="fas fa-lock-open"></i>
<span>Decoded</span>
</a>