mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
chg: [UI-Hash] add decoded type on graph
This commit is contained in:
Terrtia
parent
6d6de3ae54
commit
1fbd4109c2
4 changed files with 166 additions and 35 deletions
|
|
@ -176,7 +176,7 @@ if __name__ == '__main__':
|
|||
re.compile(regex_base64)
|
||||
|
||||
# map decoder function
|
||||
decoder_function = {'binary':binary_decoder,'hex':hex_decoder, 'base64':base64_decoder}
|
||||
decoder_function = {'binary':binary_decoder,'hexadecimal':hex_decoder, 'base64':base64_decoder}
|
||||
|
||||
hex_max_execution_time = p.config.getint("Hex", "max_execution_time")
|
||||
binary_max_execution_time = p.config.getint("Binary", "max_execution_time")
|
||||
|
|
@ -184,7 +184,7 @@ if __name__ == '__main__':
|
|||
|
||||
# list all decoder yith regex, the order is use to search content by order
|
||||
all_decoder = [ {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time},
|
||||
{'name': 'hex', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time},
|
||||
{'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time},
|
||||
{'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}]
|
||||
|
||||
for decoder in all_decoder:
|
||||
|
|
|
|||
|
|
@ -99,27 +99,40 @@ def all_hash_search():
|
|||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
type = request.form.get('type')
|
||||
print(type)
|
||||
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type))
|
||||
encoding = request.form.get('encoding')
|
||||
print(encoding)
|
||||
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding))
|
||||
|
||||
@hashDecoded.route("/hashDecoded/", methods=['GET'])
|
||||
def hashDecoded_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
type = request.args.get('type')
|
||||
encoding = request.args.get('encoding')
|
||||
|
||||
if type == 'All types':
|
||||
type = None
|
||||
|
||||
if encoding == 'All encoding':
|
||||
encoding = None
|
||||
|
||||
#date_from = '20180628' or date_from = '2018-06-28'
|
||||
#date_to = '20180628' or date_to = '2018-06-28'
|
||||
|
||||
# verify file type input
|
||||
if type is not None:
|
||||
#retrieve + char
|
||||
type = type.replace(' ', '+')
|
||||
if type not in r_serv_metadata.smembers('hash_all_type'):
|
||||
type = None
|
||||
|
||||
all_encoding = r_serv_metadata.smembers('all_decoder')
|
||||
# verify encoding input
|
||||
if encoding is not None:
|
||||
if encoding not in all_encoding:
|
||||
encoding = None
|
||||
print(encoding)
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
|
|
@ -150,7 +163,10 @@ def hashDecoded_page():
|
|||
|
||||
l_64 = set()
|
||||
for date in date_range:
|
||||
if encoding is None:
|
||||
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
|
||||
else:
|
||||
l_hash = r_serv_metadata.zrange(encoding+'_date:' +date, 0, -1)
|
||||
if l_hash:
|
||||
for hash in l_hash:
|
||||
l_64.add(hash)
|
||||
|
|
@ -199,7 +215,7 @@ def hashDecoded_page():
|
|||
l_type = r_serv_metadata.smembers('hash_all_type')
|
||||
|
||||
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
|
||||
date_from=date_from, date_to=date_to)
|
||||
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_by_type')
|
||||
def hash_by_type():
|
||||
|
|
@ -234,6 +250,14 @@ def showHash():
|
|||
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
|
||||
nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
|
||||
|
||||
# get all encoding for this hash
|
||||
list_hash_decoder = []
|
||||
list_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
for decoder in list_decoder:
|
||||
encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder')
|
||||
if encoding is not None:
|
||||
list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding})
|
||||
|
||||
num_day_type = 6
|
||||
date_range_sparkline = get_date_range(num_day_type)
|
||||
sparkline_values = list_sparkline_values(date_range_sparkline, hash)
|
||||
|
|
@ -253,7 +277,7 @@ def showHash():
|
|||
return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link,
|
||||
b64_vt_report=b64_vt_report,
|
||||
size=size, estimated_type=estimated_type, file_icon=file_icon,
|
||||
first_seen=first_seen,
|
||||
first_seen=first_seen, list_hash_decoder=list_hash_decoder,
|
||||
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
|
||||
|
||||
@app.route('/hashDecoded/downloadHash')
|
||||
|
|
@ -299,21 +323,67 @@ def hash_by_type_json():
|
|||
type = type.replace(' ', '+')
|
||||
|
||||
num_day_type = 30
|
||||
date_range_sparkline = get_date_range(num_day_type)
|
||||
date_range = get_date_range(num_day_type)
|
||||
|
||||
#verify input
|
||||
if type in r_serv_metadata.smembers('hash_all_type'):
|
||||
type_value = []
|
||||
for date in date_range_sparkline:
|
||||
all_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
|
||||
range_decoder = []
|
||||
for date in date_range:
|
||||
day_decoder = {}
|
||||
day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for decoder in all_decoder:
|
||||
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
|
||||
if num_day_decoder is None:
|
||||
num_day_decoder = 0
|
||||
day_decoder[decoder]= num_day_decoder
|
||||
range_decoder.append(day_decoder)
|
||||
|
||||
|
||||
|
||||
return jsonify(range_decoder)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
####################################
|
||||
all_type = set()
|
||||
for date in date_range:
|
||||
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
|
||||
if l_hash:
|
||||
for hash in l_hash:
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
all_type.add(estimated_type)
|
||||
|
||||
range_type = []
|
||||
|
||||
for date in date_range:
|
||||
if len(date_range) == 1:
|
||||
if date==date_from and date==date_to:
|
||||
for type in all_type:
|
||||
day_type = {}
|
||||
day_type['date']= type
|
||||
list_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
for decoder in list_decoder:
|
||||
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
|
||||
if num_day_decoder is None:
|
||||
num_day_decoder = 0
|
||||
day_type[decoder]= num_day_decoder
|
||||
range_type.append(day_type)
|
||||
else:
|
||||
range_type = ''
|
||||
else:
|
||||
day_type = {}
|
||||
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for type in all_type:
|
||||
num_day_type = r_serv_metadata.zscore('hash_type:'+type, date)
|
||||
if num_day_type is None:
|
||||
num_day_type = 0
|
||||
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
type_value.append({ 'date' : date, 'value' : int( num_day_type )})
|
||||
day_type[type]= num_day_type
|
||||
range_type.append(day_type)
|
||||
|
||||
return jsonify(type_value)
|
||||
else:
|
||||
return jsonify()
|
||||
return jsonify(range_type)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/daily_type_json')
|
||||
def daily_type_json():
|
||||
|
|
@ -358,7 +428,23 @@ def range_type_json():
|
|||
all_type.add(estimated_type)
|
||||
|
||||
range_type = []
|
||||
|
||||
for date in date_range:
|
||||
if len(date_range) == 1:
|
||||
if date==date_from and date==date_to:
|
||||
for type in all_type:
|
||||
day_type = {}
|
||||
day_type['date']= type
|
||||
list_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
for decoder in list_decoder:
|
||||
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
|
||||
if num_day_decoder is None:
|
||||
num_day_decoder = 0
|
||||
day_type[decoder]= num_day_decoder
|
||||
range_type.append(day_type)
|
||||
else:
|
||||
range_type = ''
|
||||
else:
|
||||
day_type = {}
|
||||
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for type in all_type:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,17 @@
|
|||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
||||
</div>
|
||||
Encoding :
|
||||
<select class="form-control" name="encoding" style="width=100%;">
|
||||
<option>All encoding</option>
|
||||
{% for encod in all_encoding %}
|
||||
{% if encoding|string() == encod|string() %}
|
||||
<option selected>{{ encod }}</option>
|
||||
{% else %}
|
||||
<option>{{ encod }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
File Type :
|
||||
<select class="form-control" name="type" style="width=100%;">
|
||||
<option>All types</option>
|
||||
|
|
@ -217,11 +228,11 @@
|
|||
});
|
||||
|
||||
{% if type %}
|
||||
chart.stackBarChart =barchart_type('/hashDecoded/hash_by_type_json?type={{type}}', '#barchart_type');
|
||||
chart.stackBarChart =barchart_type_stack('/hashDecoded/hash_by_type_json?type={{type}}', 'id');
|
||||
{% elif daily_type_chart %}
|
||||
chart.stackBarChart =barchart_type('/hashDecoded/daily_type_json?date={{daily_date}}', '#barchart_type');
|
||||
chart.stackBarChart =barchart_type_stack('/hashDecoded/range_type_json?date_from={{daily_date}}&date_to={{daily_date}}', 'id');
|
||||
{% else %}
|
||||
chart.stackBarChart = barchart_type_stack('url', 'id')
|
||||
chart.stackBarChart = barchart_type_stack("/hashDecoded/range_type_json?date_from={{date_from}}&date_to={{date_to}}", 'id')
|
||||
{% endif %}
|
||||
|
||||
chart.onResize();
|
||||
|
|
@ -294,7 +305,7 @@ function sparklines(id, points) {
|
|||
</script>
|
||||
|
||||
<script>
|
||||
var margin = {top: 20, right: 55, bottom: 30, left: 40},
|
||||
var margin = {top: 20, right: 55, bottom: 55, left: 40},
|
||||
width = 1000 - margin.left - margin.right,
|
||||
height = 500 - margin.top - margin.bottom;
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
|
|
@ -317,7 +328,7 @@ var svg = d3.select("#barchart_type").append("svg")
|
|||
|
||||
function barchart_type_stack(url, id) {
|
||||
|
||||
d3.json("/hashDecoded/range_type_json?date_from={{date_from}}&date_to={{date_to}}")
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
var labelVar = 'date'; //A
|
||||
|
|
@ -346,9 +357,17 @@ function barchart_type_stack(url, id) {
|
|||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.attr("class", "bar")
|
||||
.on("click", function (d) { window.location.href = "/hashDecoded/"+'?date_from='+d+'&date_to='+d })
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "rotate(-45)" );
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from={{date_from}}&date_to={{date_to}}&type='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% else %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-40)" )
|
||||
{% endif %}
|
||||
.style("text-anchor", "end");
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
|
|
@ -376,7 +395,13 @@ function barchart_type_stack(url, id) {
|
|||
.style("stroke", "grey")
|
||||
.on("mouseover", function (d) { showPopover.call(this, d); })
|
||||
.on("mouseout", function (d) { removePopovers(); })
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?date_from={{date_from}}&date_to={{date_to}}&type='+d.label+'&encoding='+d.name; });
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type={{type}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
|
||||
{% else %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
|
||||
{% endif %}
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.total != 0){
|
||||
|
|
@ -386,7 +411,13 @@ function barchart_type_stack(url, id) {
|
|||
//.on("click", (window.location.href = "/hashDecoded/"+'?date_from='+d.date) )
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.total))
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from={{date_from}}&date_to={{date_to}}&type='+d.date })
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function () {window.location.href = '/hashDecoded/?type={{type}}&date_from='+d.date+'&date_to='+d.date })
|
||||
{% else %}
|
||||
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from='+d.date+'&date_to='+d.date })
|
||||
{% endif %}
|
||||
.style("text-anchor", "middle")
|
||||
.text(d.total);
|
||||
}
|
||||
|
|
@ -434,7 +465,7 @@ function showPopover (d) {
|
|||
trigger: 'manual',
|
||||
html : true,
|
||||
content: function() {
|
||||
return "date: " + d.label +
|
||||
return d.label +
|
||||
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
|
||||
});
|
||||
$(this).popover('show')
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@
|
|||
<th>First_seen</th>
|
||||
<th>Last_seen</th>
|
||||
<th>Size (Kb)</th>
|
||||
<th>nb_seen_in_all_pastes</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
|
@ -172,6 +172,20 @@
|
|||
</div>
|
||||
|
||||
<div class="col-md-2">
|
||||
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-unlock-alt" aria-hidden="true"></i> Encoding
|
||||
</div>
|
||||
<div class="panel-body" style="text-align:center;">
|
||||
{% for encoding in list_hash_decoder %}
|
||||
<button id="" class="btn btn-default">
|
||||
{{encoding['encoding']}} <span class="badge">{{encoding['nb_seen']}}</span>
|
||||
</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
|
||||
|
|
|
|||
Loading…
Reference in a new issue