mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
new: add base64 display on paste + base64 statistics
This commit is contained in:
Terrtia
parent
a2931a18fc
commit
16c700ddb4
18 changed files with 1008 additions and 27 deletions
31
OVERVIEW.md
31
OVERVIEW.md
|
|
@ -1,7 +1,7 @@
|
|||
Overview
|
||||
========
|
||||
|
||||
Redis and LevelDB overview
|
||||
Redis and ARDB overview
|
||||
--------------------------
|
||||
|
||||
* Redis on TCP port 6379
|
||||
|
|
@ -20,3 +20,32 @@ Redis and LevelDB overview
|
|||
- DB 0 - Lines duplicate
|
||||
- DB 1 - Hashs
|
||||
|
||||
|
||||
ARDB overview
|
||||
---------------------------
|
||||
|
||||
* DB 7 - Metadata:
|
||||
----------------------------------------- BASE64 ----------------------------------------
|
||||
|
||||
HSET - 'metadata_hash:'+hash 'saved_path' saved_path
|
||||
'size' size
|
||||
'first_seen' first_seen
|
||||
'last_seen' last_seen
|
||||
'estimated_type' estimated_type
|
||||
'vt_link' vt_link
|
||||
'vt_report' vt_report
|
||||
'nb_seen_in_all_pastes' nb_seen_in_all_pastes
|
||||
|
||||
SET - 'base64_paste:'+paste hash *
|
||||
|
||||
ZADD - 'base64_date:'+20180622 hash * nb_seen_this_day
|
||||
|
||||
ZADD - 'base64_hash'+hash paste * nb_seen_in_paste
|
||||
|
||||
SET - 'hash_all_type' hash_type
|
||||
|
||||
ZADD - 'base64_type:'+type date nb_seen
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -86,8 +86,7 @@ if __name__ == "__main__":
|
|||
|
||||
if message is not None:
|
||||
|
||||
search_api_key(message)
|
||||
|
||||
search_api_key(message)
|
||||
|
||||
else:
|
||||
publisher.debug("Script ApiKey is Idling 10s")
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import time
|
||||
import os
|
||||
import datetime
|
||||
import redis
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ def timeout_handler(signum, frame):
|
|||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
|
||||
def search_base64(content, message):
|
||||
def search_base64(content, message, date):
|
||||
find = False
|
||||
base64_list = re.findall(regex_base64, content)
|
||||
if(len(base64_list) > 0):
|
||||
|
|
@ -46,6 +47,8 @@ def search_base64(content, message):
|
|||
|
||||
find = True
|
||||
hash = sha1(decode).hexdigest()
|
||||
print(message)
|
||||
print(hash)
|
||||
|
||||
data = {}
|
||||
data['name'] = hash
|
||||
|
|
@ -54,8 +57,36 @@ def search_base64(content, message):
|
|||
data['estimated type'] = type
|
||||
json_data = json.dumps(data)
|
||||
|
||||
save_base64_as_file(decode, type, hash, json_data)
|
||||
print('found {} '.format(type))
|
||||
date_paste = '{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
date_key = date[0:4] + date[4:6] + date[6:8]
|
||||
|
||||
serv_metadata.zincrby('base64_date:'+date_key, hash, 1)
|
||||
|
||||
# first time we see this hash
|
||||
if not serv_metadata.hexists('metadata_hash:'+hash, 'estimated_type'):
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'first_seen', date_paste)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
else:
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
|
||||
# first time we see this file on this paste
|
||||
if serv_metadata.zscore('base64_hash:'+hash, message) is None:
|
||||
print('first')
|
||||
serv_metadata.hincrby('metadata_hash:'+hash, 'nb_seen_in_all_pastes', 1)
|
||||
|
||||
serv_metadata.sadd('base64_paste:'+message, hash) # paste - hash map
|
||||
serv_metadata.zincrby('base64_hash:'+hash, message, 1)# hash - paste map
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'estimated_type', type)
|
||||
serv_metadata.sadd('hash_all_type', type)
|
||||
serv_metadata.zincrby('base64_type:'+type, date_key, 1)
|
||||
|
||||
save_base64_as_file(decode, type, hash, json_data, id)
|
||||
print('found {} '.format(type))
|
||||
# duplicate
|
||||
else:
|
||||
serv_metadata.zincrby('base64_hash:'+hash, message, 1) # number of b64 on this paste
|
||||
|
||||
if(find):
|
||||
publisher.warning('base64 decoded')
|
||||
|
|
@ -68,10 +99,10 @@ def search_base64(content, message):
|
|||
msg = 'infoleak:automatic-detection="base64";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
def save_base64_as_file(decode, type, hash, json_data):
|
||||
def save_base64_as_file(decode, type, hash, json_data, id):
|
||||
|
||||
filename_b64 = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "base64"), type, hash[:2], hash)
|
||||
local_filename_b64 = os.path.join(p.config.get("Directories", "base64"), type, hash[:2], hash)
|
||||
filename_b64 = os.path.join(os.environ['AIL_HOME'], local_filename_b64)
|
||||
|
||||
filename_json = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "base64"), type, hash[:2], hash + '.json')
|
||||
|
|
@ -83,6 +114,10 @@ def save_base64_as_file(decode, type, hash, json_data):
|
|||
with open(filename_b64, 'wb') as f:
|
||||
f.write(decode)
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'saved_path', local_filename_b64)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'size', os.path.getsize(filename_b64))
|
||||
|
||||
with open(filename_json, 'w') as f:
|
||||
f.write(json_data)
|
||||
|
||||
|
|
@ -103,6 +138,12 @@ if __name__ == '__main__':
|
|||
p = Process(config_section)
|
||||
max_execution_time = p.config.getint("Base64", "max_execution_time")
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Base64 started")
|
||||
|
||||
|
|
@ -127,13 +168,12 @@ if __name__ == '__main__':
|
|||
# Do something with the message from the queue
|
||||
#print(filename)
|
||||
content = paste.get_p_content()
|
||||
search_base64(content,message)
|
||||
|
||||
# (Optional) Send that thing to the next queue
|
||||
#p.populate_set_out(something_has_been_done)
|
||||
date = str(paste._get_p_date())
|
||||
search_base64(content,message, date)
|
||||
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
|
|
|||
|
|
@ -135,6 +135,12 @@ class Process(object):
|
|||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.serv_statistics = redis.StrictRedis(
|
||||
host=self.config.get('ARDB_Statistics', 'host'),
|
||||
port=self.config.get('ARDB_Statistics', 'port'),
|
||||
db=self.config.get('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.moduleNum = os.getpid()
|
||||
|
||||
def populate_set_in(self):
|
||||
|
|
@ -181,6 +187,9 @@ class Process(object):
|
|||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
|
||||
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
|
||||
return message
|
||||
|
||||
#except:
|
||||
|
|
@ -217,3 +226,7 @@ class Process(object):
|
|||
time.sleep(1)
|
||||
continue
|
||||
self.pubsub.publish(message)
|
||||
|
||||
def incr_module_timeout_statistic(self):
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)
|
||||
|
|
|
|||
|
|
@ -155,6 +155,8 @@ if __name__ == "__main__":
|
|||
flag_the_hive = False
|
||||
r_serv_db.set('ail:thehive', False)
|
||||
print('Not connected to The HIVE')
|
||||
else:
|
||||
HiveApi = False
|
||||
|
||||
if HiveApi != False and flag_the_hive:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -61,7 +61,8 @@ if __name__ == "__main__":
|
|||
publisher.info(to_print)
|
||||
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
Analyse(message, server)
|
||||
except TimeoutException:
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(message))
|
||||
continue
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ This module create tags.
|
|||
import redis
|
||||
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
|
@ -41,6 +42,12 @@ if __name__ == '__main__':
|
|||
db=p.config.get("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
serv_statistics = redis.StrictRedis(
|
||||
host=p.config.get('ARDB_Statistics', 'host'),
|
||||
port=p.config.get('ARDB_Statistics', 'port'),
|
||||
db=p.config.get('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Tags module started")
|
||||
|
||||
|
|
@ -67,4 +74,6 @@ if __name__ == '__main__':
|
|||
print(" tagged: {}".format(tag))
|
||||
server_metadata.sadd('tag:'+path, tag)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_tagged:'+tag, 1)
|
||||
p.populate_set_out(message, 'MISP_The_Hive_feeder')
|
||||
|
|
|
|||
|
|
@ -60,8 +60,9 @@ if __name__ == "__main__":
|
|||
msg = '{} {} {}'.format(paste.p_path, word, score)
|
||||
p.populate_set_out(msg)
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class Paste(object):
|
|||
|
||||
:Example:
|
||||
|
||||
PST = Paste("/home/2013/ZEeGaez5.gz")
|
||||
PST = Paste("/home/2013/01/12/ZEeGaez5.gz")
|
||||
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,9 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
|
|||
print(' {} send to Global'.format(save_path))
|
||||
r_serv_log_submit.sadd(uuid + ':paste_submit_link', full_path)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_paste', 1)
|
||||
|
||||
return 0
|
||||
|
||||
def addError(uuid, errorMessage):
|
||||
|
|
@ -60,6 +63,8 @@ def addError(uuid, errorMessage):
|
|||
def abord_file_submission(uuid, errorMessage):
|
||||
addError(uuid, errorMessage)
|
||||
r_serv_log_submit.set(uuid + ':end', 1)
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1)
|
||||
remove_submit_uuid(uuid)
|
||||
|
||||
|
||||
|
|
@ -154,6 +159,12 @@ if __name__ == "__main__":
|
|||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
serv_statistics = redis.StrictRedis(
|
||||
host=cfg.get('ARDB_Statistics', 'host'),
|
||||
port=cfg.getint('ARDB_Statistics', 'port'),
|
||||
db=cfg.getint('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
expire_time = 120
|
||||
MAX_FILE_SIZE = 1000000000
|
||||
ALLOWED_EXTENSIONS = ['txt', 'sh', 'pdf']
|
||||
|
|
|
|||
|
|
@ -143,3 +143,18 @@ DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display t
|
|||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||
|
||||
# VT
|
||||
try:
|
||||
from virusTotalKEYS import vt_key
|
||||
if vt_key != '':
|
||||
vt_auth = vt_key
|
||||
vt_enabled = True
|
||||
print('VT submission is enabled')
|
||||
else:
|
||||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
except:
|
||||
vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")}
|
||||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
|
|
|
|||
248
var/www/modules/base64Decoded/Flask_base64Decoded.py
Normal file
248
var/www/modules/base64Decoded/Flask_base64Decoded.py
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
'''
|
||||
import redis
|
||||
import os
|
||||
import datetime
|
||||
import json
|
||||
from Date import Date
|
||||
import requests
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
vt_auth = Flask_config.vt_auth
|
||||
|
||||
base64Decoded = Blueprint('base64Decoded', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def get_date_range(num_day):
|
||||
curr_date = datetime.date.today()
|
||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
date_list = []
|
||||
|
||||
for i in range(0, num_day+1):
|
||||
date_list.append(date.substract_day(i))
|
||||
|
||||
return list(reversed(date_list))
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
def one():
|
||||
return 1
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@base64Decoded.route("/base64Decoded/", methods=['GET'])
|
||||
def base64Decoded_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
type = request.args.get('type')
|
||||
|
||||
#date_from = '20180628'
|
||||
#date_to = '20180628'
|
||||
|
||||
if type not in r_serv_metadata.smembers('hash_all_type'):
|
||||
type = None
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
|
||||
# display day type bar chart
|
||||
if len(date_range) == 1 and type is None:
|
||||
daily_type_chart = True
|
||||
daily_date = date_range[0]
|
||||
else:
|
||||
daily_type_chart = False
|
||||
daily_date = None
|
||||
|
||||
l_64 = set()
|
||||
for date in date_range:
|
||||
l_hash = r_serv_metadata.zrange('base64_date:' +date, 0, -1)
|
||||
if l_hash:
|
||||
for hash in l_hash:
|
||||
l_64.add(hash)
|
||||
|
||||
num_day_sparkline = 6
|
||||
date_range_sparkline = get_date_range(num_day_sparkline)
|
||||
|
||||
b64_metadata = []
|
||||
l_64 = list(l_64)
|
||||
for hash in l_64:
|
||||
# select requested base 64 type
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
if type is not None:
|
||||
if estimated_type is not None:
|
||||
if estimated_type != type:
|
||||
continue
|
||||
|
||||
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
|
||||
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
|
||||
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
|
||||
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
|
||||
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
|
||||
if hash is not None and first_seen is not None and \
|
||||
last_seen is not None and \
|
||||
nb_seen_in_paste is not None and \
|
||||
size is not None:
|
||||
|
||||
file_type = estimated_type.split('/')[0]
|
||||
# set file icon
|
||||
if file_type == 'application':
|
||||
file_icon = 'fa-file-o '
|
||||
elif file_type == 'audio':
|
||||
file_icon = 'fa-file-video-o '
|
||||
elif file_type == 'image':
|
||||
file_icon = 'fa-file-image-o'
|
||||
elif file_type == 'text':
|
||||
file_icon = 'fa-file-text-o'
|
||||
else:
|
||||
file_icon = 'fa-file'
|
||||
|
||||
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
|
||||
b64_vt = True
|
||||
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
|
||||
else:
|
||||
b64_vt = False
|
||||
b64_vt_link = ''
|
||||
|
||||
sparklines_value = []
|
||||
for date_day in date_range_sparkline:
|
||||
nb_seen_this_day = r_serv_metadata.zscore('base64_date:'+date_day, hash)
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
sparklines_value.append(int(nb_seen_this_day))
|
||||
|
||||
b64_metadata.append( (file_icon, estimated_type, hash, nb_seen_in_paste, size, first_seen, last_seen, b64_vt, b64_vt_link, sparklines_value) )
|
||||
|
||||
return render_template("base64Decoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date)
|
||||
|
||||
@base64Decoded.route('/base64Decoded/hash_by_type')
|
||||
def hash_by_type():
|
||||
type = request.args.get('type')
|
||||
type = 'text/plain'
|
||||
return render_template('base64_type.html',type = type)
|
||||
|
||||
@base64Decoded.route('/base64Decoded/hash_by_type_json')
|
||||
def hash_by_type_json():
|
||||
type = request.args.get('type')
|
||||
|
||||
num_day_type = 30
|
||||
date_range_sparkline = get_date_range(num_day_type)
|
||||
|
||||
#verify input
|
||||
if type in r_serv_metadata.smembers('hash_all_type'):
|
||||
type_value = []
|
||||
for date in date_range_sparkline:
|
||||
num_day_type = r_serv_metadata.zscore('base64_type:'+type, date)
|
||||
if num_day_type is None:
|
||||
num_day_type = 0
|
||||
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
type_value.append({ 'date' : date, 'value' : int( num_day_type )})
|
||||
|
||||
return jsonify(type_value)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
@base64Decoded.route('/base64Decoded/daily_type_json')
|
||||
def daily_type_json():
|
||||
date = request.args.get('date')
|
||||
|
||||
daily_type = set()
|
||||
l_b64 = r_serv_metadata.zrange('base64_date:' +date, 0, -1)
|
||||
for hash in l_b64:
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
if estimated_type is not None:
|
||||
daily_type.add(estimated_type)
|
||||
|
||||
type_value = []
|
||||
for day_type in daily_type:
|
||||
num_day_type = r_serv_metadata.zscore('base64_type:'+day_type, date)
|
||||
type_value.append({ 'date' : day_type, 'value' : int( num_day_type )})
|
||||
return jsonify(type_value)
|
||||
|
||||
@base64Decoded.route('/base64Decoded/send_file_to_vt', methods=['POST'])
|
||||
def send_file_to_vt():
|
||||
paste = request.form['paste']
|
||||
hash = request.form['hash']
|
||||
|
||||
b64_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
|
||||
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
|
||||
b64_content = ''
|
||||
with open(b64_full_path, 'rb') as f:
|
||||
b64_content = f.read()
|
||||
|
||||
files = {'file': (hash, b64_content)}
|
||||
response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params= {'apikey': vt_auth})
|
||||
json_response = response.json()
|
||||
print(json_response)
|
||||
|
||||
vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link)
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report', '')
|
||||
|
||||
return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste))
|
||||
|
||||
@base64Decoded.route('/base64Decoded/update_vt_result')
|
||||
def update_vt_result():
|
||||
hash = request.args.get('hash')
|
||||
|
||||
params = {'apikey': vt_auth, 'resource': hash}
|
||||
response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',params=params)
|
||||
if response.status_code == 200:
|
||||
json_response = response.json()
|
||||
response_code = json_response['response_code']
|
||||
# report exist
|
||||
if response_code == 1:
|
||||
total = json_response['total']
|
||||
positive = json_response['positives']
|
||||
|
||||
b64_vt_report = 'Detection {}/{}'.format(positive,total)
|
||||
# no report found
|
||||
elif response_code == 0:
|
||||
b64_vt_report = 'No report found'
|
||||
pass
|
||||
# file in queue
|
||||
elif response_code == -2:
|
||||
b64_vt_report = 'File in queue'
|
||||
pass
|
||||
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_report', b64_vt_report)
|
||||
return jsonify(hash=hash, report_vt=b64_vt_report)
|
||||
elif response.status_code == 403:
|
||||
Flask_config.vt_enabled = False
|
||||
print('VT is disabled')
|
||||
return jsonify()
|
||||
else:
|
||||
# TODO FIXME make json response
|
||||
return jsonify()
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(base64Decoded)
|
||||
306
var/www/modules/base64Decoded/templates/base64Decoded.html
Normal file
306
var/www/modules/base64Decoded/templates/base64Decoded.html
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.js') }}"></script>
|
||||
<style>
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Base64 Files</h1>
|
||||
<div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
|
||||
{% if type %}
|
||||
<div id="barchart_type">
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if daily_type_chart %}
|
||||
<div id="barchart_type">
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- /#page-wrapper -->
|
||||
{% if l_64|length != 0 %}
|
||||
<h3> Today Base64 files: </h3>
|
||||
<table id="tableb64" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>estimated type</th>
|
||||
<th>hash</th>
|
||||
<th>first seen</th>
|
||||
<th>last seen</th>
|
||||
<th>nb paste</th>
|
||||
<th>size</th>
|
||||
<th>Virus Total</th>
|
||||
<th>Test</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fa {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td>{{ b64[2] }}</td>
|
||||
<td>{{ b64[5] }}</td>
|
||||
<td>{{ b64[6] }}</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td>{{ b64[4] }}</td>
|
||||
<td style="text-align:center;max-width:150px;">
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[7] %}
|
||||
<form action="/base64Decoded/send_file_to_vt" id="vt_submit" method='post'>
|
||||
<input type="hidden" id="hash" name="hash" value="b64[2]">
|
||||
<button class="btn btn-primary">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</form>
|
||||
{% else %}
|
||||
<a class="btn btn-default" target="_blank" href="{{ b64[8] }}"><i class="fa fa-link"> VT scan reports</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
|
||||
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span> {{ b64[8] }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
</td>
|
||||
<td id="sparklines_{{ b64[2] }}" style="text-align:center;">
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<h3>{{daily_date[0:4]}}/{{daily_date[4:6]}}/{{daily_date[6:8]}}, No base64</h3>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
activePage = "page-base64Decoded"
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
|
||||
{% if type %}
|
||||
barchart_type('/base64Decoded/hash_by_type_json?type={{type}}', '#barchart_type');
|
||||
{% endif %}
|
||||
{% if daily_type_chart %}
|
||||
barchart_type('/base64Decoded/daily_type_json?date={{daily_date}}', '#barchart_type');
|
||||
console.log('cool')
|
||||
{% endif %}
|
||||
|
||||
});
|
||||
</script>
|
||||
<script>
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON('/base64Decoded/update_vt_result?hash='+hash,
|
||||
function(data) {
|
||||
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
|
||||
$( "#report_vt_"+hash ).html(content);
|
||||
});
|
||||
}
|
||||
|
||||
</script>
|
||||
<script>
|
||||
//var data = [6,3,3,2,5,3,9];
|
||||
|
||||
// a sparklines plot
|
||||
function sparklines(id, points) {
|
||||
var width = 100, height = 60;
|
||||
|
||||
var data = []
|
||||
for (i = 0; i < points.length; i++) {
|
||||
data[i] = {
|
||||
'x': i,
|
||||
'y': +points[i]
|
||||
}
|
||||
}
|
||||
|
||||
var x = d3.scaleLinear()
|
||||
.range([0, width - 10])
|
||||
.domain([0,5]);
|
||||
|
||||
var y = d3.scaleLinear()
|
||||
.range([height, 0])
|
||||
.domain([0,10]);
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {return x(d.x)})
|
||||
.y(function(d) {return y(d.y)});
|
||||
|
||||
d3.select("#"+id).append('svg')
|
||||
.attr('width', width)
|
||||
.attr('height', height)
|
||||
.append('path')
|
||||
.attr('class','line')
|
||||
.datum(data)
|
||||
.attr('d', line);
|
||||
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
{% for b64 in l_64 %}
|
||||
sparklines("sparklines_{{ b64[2] }}", {{ b64[9] }})
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function barchart_type(url, id) {
|
||||
|
||||
|
||||
var margin = {top: 20, right: 20, bottom: 70, left: 40};
|
||||
|
||||
var width = 960 - margin.left - margin.right;
|
||||
var height = 500 - margin.top - margin.bottom;
|
||||
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x)
|
||||
//.tickFormat(d3.time.format("%Y-%m"));
|
||||
|
||||
var yAxis = d3.axisLeft(y)
|
||||
.ticks(10);
|
||||
|
||||
var svg = d3.select(id).append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
{% if daily_type_chart %}
|
||||
x.domain(data.map(function(d) { return d.date; }));
|
||||
{% else %}
|
||||
x.domain(data.map(function(d) { return d.date.substring(5); }));
|
||||
{% endif %}
|
||||
y.domain([0, d3.max(data, function(d) { return d.value; })]);
|
||||
|
||||
var label = svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", "-.55em")
|
||||
{% if daily_type_chart %}
|
||||
.attr("transform", "rotate(-20)" );
|
||||
{% else %}
|
||||
.attr("transform", "rotate(-70)" );
|
||||
{% endif %}
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end")
|
||||
.text("Value ($)");
|
||||
|
||||
var bar = svg.selectAll("bar")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar")
|
||||
//.style("fill", "steelblue")
|
||||
{% if daily_type_chart %}
|
||||
.attr("x", function(d) { return x(d.date); })
|
||||
{% else %}
|
||||
.attr("x", function(d) { return x(d.date.substring(5)); })
|
||||
{% endif %}
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function(d) { return y(d.value); })
|
||||
.attr("height", function(d) { return height - y(d.value); })
|
||||
{% if type %}
|
||||
.on("click", function(d){ window.location.href = "/base64Decoded/" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
|
||||
{% endif %}
|
||||
{% if daily_type_chart %}
|
||||
.on("click", function(d){ window.location.href = "/base64Decoded/" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
|
||||
{% endif %}
|
||||
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.value != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.text(function(d) { return d.value; });
|
||||
.text(d.value)
|
||||
{% if daily_type_chart %}
|
||||
.attr('x', x(d.date) + x.bandwidth()/4)
|
||||
{% else %}
|
||||
.attr('x', x(d.date.substring(5)) + x.bandwidth()/4)
|
||||
{% endif %}
|
||||
.attr('y', y(d.value));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
179
var/www/modules/base64Decoded/templates/base64_type.html
Normal file
179
var/www/modules/base64Decoded/templates/base64_type.html
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.js') }}"></script>
|
||||
<style>
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
}
|
||||
|
||||
.bar span {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.bar span .tooltiptext {
|
||||
visibility: hidden;
|
||||
width: 120px;
|
||||
background-color: #555;
|
||||
color: #fff;
|
||||
text-align: center;
|
||||
border-radius: 6px;
|
||||
padding: 5px 0;
|
||||
position: absolute;
|
||||
z-index: 1;
|
||||
bottom: 125%;
|
||||
left: 50%;
|
||||
margin-left: -60px;
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s;
|
||||
}
|
||||
|
||||
|
||||
.bar span:hover .tooltiptext {
|
||||
visibility: visible;
|
||||
opacity: 1;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Base64 Files</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
|
||||
<div id="barchart_type">
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
activePage = "page-base64Decoded"
|
||||
$("#"+activePage).addClass("active");
|
||||
barchart_type('url', 'id')
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function barchart_type(url, id) {
|
||||
|
||||
|
||||
var margin = {top: 20, right: 20, bottom: 70, left: 40};
|
||||
|
||||
var width = 960 - margin.left - margin.right;
|
||||
var height = 500 - margin.top - margin.bottom;
|
||||
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x)
|
||||
//.tickFormat(d3.time.format("%Y-%m"));
|
||||
|
||||
var yAxis = d3.axisLeft(y)
|
||||
.ticks(10);
|
||||
|
||||
var svg = d3.select("#barchart_type").append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
|
||||
d3.json("/base64Decoded/hash_by_type_json?type={{type}}")
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
x.domain(data.map(function(d) { return d.date.substring(5); }));
|
||||
y.domain([0, d3.max(data, function(d) { return d.value; })]);
|
||||
|
||||
var label = svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", "-.55em")
|
||||
.attr("transform", "rotate(-70)" );
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end")
|
||||
.text("Value ($)");
|
||||
|
||||
var bar = svg.selectAll("bar")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar")
|
||||
//.style("fill", "steelblue")
|
||||
.attr("x", function(d) { return x(d.date.substring(5)); })
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function(d) { return y(d.value); })
|
||||
.attr("height", function(d) { return height - y(d.value); })
|
||||
.on("click", function(d){ window.location.href = "/base64Decoded/" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
|
||||
|
||||
bar.append("span")
|
||||
.attr("class", "tooltiptext")
|
||||
.text('bonjour');
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.value != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.text(function(d) { return d.value; });
|
||||
.text(d.value)
|
||||
.attr('x', x(d.date.substring(5)) + x.bandwidth()/4)
|
||||
.attr('y', y(d.value));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1 @@
|
|||
<li id='page-base64Decoded'><a href="{{ url_for('base64Decoded.base64Decoded_page') }}"><i class="glyphicon glyphicon-new-window"></i> base64Decoded </a></li>
|
||||
|
|
@ -6,12 +6,14 @@
|
|||
'''
|
||||
import redis
|
||||
import json
|
||||
import os
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, make_response
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, make_response, redirect, url_for
|
||||
import difflib
|
||||
import ssdeep
|
||||
|
||||
import Paste
|
||||
import requests
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
|
@ -28,13 +30,15 @@ DiffMaxLineLength = Flask_config.DiffMaxLineLength
|
|||
bootstrap_label = Flask_config.bootstrap_label
|
||||
misp_event_url = Flask_config.misp_event_url
|
||||
hive_case_url = Flask_config.hive_case_url
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
|
||||
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def showpaste(content_range):
|
||||
requested_path = request.args.get('paste', '')
|
||||
def showpaste(content_range, requested_path):
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
|
||||
paste = Paste.Paste(requested_path)
|
||||
p_date = str(paste._get_p_date())
|
||||
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
|
||||
|
|
@ -118,7 +122,6 @@ def showpaste(content_range):
|
|||
else:
|
||||
automatic = False
|
||||
|
||||
tag_hash = ssdeep.hash(tag)
|
||||
if r_serv_statistics.sismember('tp:'+tag, requested_path):
|
||||
tag_status_tp = True
|
||||
else:
|
||||
|
|
@ -130,6 +133,37 @@ def showpaste(content_range):
|
|||
|
||||
list_tags.append( (tag, automatic, tag_status_tp, tag_status_fp) )
|
||||
|
||||
l_64 = []
|
||||
# load base64 files
|
||||
if r_serv_metadata.scard('base64_paste:'+requested_path) > 0:
|
||||
set_b64 = r_serv_metadata.smembers('base64_paste:'+requested_path)
|
||||
for hash in set_b64:
|
||||
nb_in_file = int(r_serv_metadata.zscore('base64_hash:'+hash, requested_path))
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
file_type = estimated_type.split('/')[0]
|
||||
# set file icon
|
||||
if file_type == 'application':
|
||||
file_icon = 'fa-file-o '
|
||||
elif file_type == 'audio':
|
||||
file_icon = 'fa-file-video-o '
|
||||
elif file_type == 'image':
|
||||
file_icon = 'fa-file-image-o'
|
||||
elif file_type == 'text':
|
||||
file_icon = 'fa-file-text-o'
|
||||
else:
|
||||
file_icon = 'fa-file'
|
||||
saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
|
||||
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
|
||||
b64_vt = True
|
||||
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
|
||||
print(b64_vt_report)
|
||||
else:
|
||||
b64_vt = False
|
||||
b64_vt_link = ''
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
|
||||
l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) )
|
||||
|
||||
if Flask_config.pymisp is False:
|
||||
misp = False
|
||||
else:
|
||||
|
|
@ -157,13 +191,15 @@ def showpaste(content_range):
|
|||
hive_url = hive_case_url.replace('id_here', hive_case)
|
||||
|
||||
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list,
|
||||
misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
|
||||
l_64=l_64, vt_enabled=vt_enabled, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
|
||||
def showsavedpaste():
|
||||
return showpaste(0)
|
||||
requested_path = request.args.get('paste', '')
|
||||
print(requested_path)
|
||||
return showpaste(0, requested_path)
|
||||
|
||||
@showsavedpastes.route("/showsavedrawpaste/") #shows raw
|
||||
def showsavedrawpaste():
|
||||
|
|
@ -175,7 +211,8 @@ def showsavedrawpaste():
|
|||
@showsavedpastes.route("/showpreviewpaste/")
|
||||
def showpreviewpaste():
|
||||
num = request.args.get('num', '')
|
||||
return "|num|"+num+"|num|"+showpaste(max_preview_modal)
|
||||
requested_path = request.args.get('paste', '')
|
||||
return "|num|"+num+"|num|"+showpaste(max_preview_modal, requested_path)
|
||||
|
||||
|
||||
@showsavedpastes.route("/getmoredata/")
|
||||
|
|
@ -202,5 +239,26 @@ def showDiff():
|
|||
the_html = htmlD.make_file(lines1, lines2)
|
||||
return the_html
|
||||
|
||||
@showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
|
||||
def send_file_to_vt():
|
||||
b64_path = request.form['b64_path']
|
||||
paste = request.form['paste']
|
||||
hash = request.form['hash']
|
||||
|
||||
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
|
||||
b64_content = ''
|
||||
with open(b64_full_path, 'rb') as f:
|
||||
b64_content = f.read()
|
||||
|
||||
files = {'file': (hash, b64_content)}
|
||||
response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=vt_auth)
|
||||
json_response = response.json()
|
||||
print(json_response)
|
||||
|
||||
vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link)
|
||||
|
||||
return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste))
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(showsavedpastes)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@
|
|||
overflow-x: hidden;
|
||||
width:100%;
|
||||
}
|
||||
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
|
@ -372,6 +377,52 @@
|
|||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% if l_64|length != 0 %}
|
||||
<h3> Base64 files: </h3>
|
||||
<table id="tableb64" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>estimated type</th>
|
||||
<th>hash</th>
|
||||
<th>saved_path</th>
|
||||
<th>Virus Total</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fa {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td>{{ b64[2] }} ({{ b64[4] }})</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td style="text-align:center;">
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[5] %}
|
||||
<form action="/base64Decoded/send_file_to_vt" id="vt_submit" method='post'>
|
||||
<input type="hidden" id="hash" name="hash" value="{{ b64[2] }}">
|
||||
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
|
||||
<button class="btn btn-primary">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</form>
|
||||
{% else %}
|
||||
<a class="btn btn-default" target="_blank" href="{{ b64[6] }}"><i class="fa fa-link"> VT scan reports</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
|
||||
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span> {{ b64[7] }}</div>
|
||||
</button>
|
||||
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<h3> Content: </h3>
|
||||
<a href="{{ url_for('showsavedpastes.showsavedrawpaste') }}?paste={{ request.args.get('paste') }}" id='raw_paste' > [Raw content] </a>
|
||||
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
|
||||
|
|
@ -406,9 +457,26 @@
|
|||
});
|
||||
|
||||
$('#tableDup').DataTable();
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 5,
|
||||
"order": [[ 1, "asc" ]]
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON('/base64Decoded/update_vt_result?hash='+hash,
|
||||
function(data) {
|
||||
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
|
||||
$( "#report_vt_"+hash ).html(content);
|
||||
});
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
jQuery("#all-tags-taxonomies").click(function(e){
|
||||
//change input tags list
|
||||
|
|
|
|||
Loading…
Reference in a new issue