AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [UI-Hash] add decoded type on graph

Browse source
This commit is contained in:
Terrtia 2018-07-20 15:48:56 +02:00
parent 6d6de3ae54
commit 1fbd4109c2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 166 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/Decoder.py
View file

@ -176,7 +176,7 @@ if __name__ == '__main__':
re.compile(regex_base64) re.compile(regex_base64)
# map decoder function # map decoder function
decoder_function = {'binary':binary_decoder,'hex':hex_decoder, 'base64':base64_decoder} decoder_function = {'binary':binary_decoder,'hexadecimal':hex_decoder, 'base64':base64_decoder}
hex_max_execution_time = p.config.getint("Hex", "max_execution_time") hex_max_execution_time = p.config.getint("Hex", "max_execution_time")
binary_max_execution_time = p.config.getint("Binary", "max_execution_time") binary_max_execution_time = p.config.getint("Binary", "max_execution_time")
@ -184,7 +184,7 @@ if __name__ == '__main__':
# list all decoder yith regex, the order is use to search content by order # list all decoder yith regex, the order is use to search content by order
all_decoder = [ {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}, all_decoder = [ {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time},
{'name': 'hex', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}, {'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time},
{'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}] {'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}]
for decoder in all_decoder: for decoder in all_decoder:

108
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -99,27 +99,40 @@ def all_hash_search():
date_from = request.form.get('date_from') date_from = request.form.get('date_from')
date_to = request.form.get('date_to') date_to = request.form.get('date_to')
type = request.form.get('type') type = request.form.get('type')
print(type) encoding = request.form.get('encoding')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type)) print(encoding)
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding))
@hashDecoded.route("/hashDecoded/", methods=['GET']) @hashDecoded.route("/hashDecoded/", methods=['GET'])
def hashDecoded_page(): def hashDecoded_page():
date_from = request.args.get('date_from') date_from = request.args.get('date_from')
date_to = request.args.get('date_to') date_to = request.args.get('date_to')
type = request.args.get('type') type = request.args.get('type')
encoding = request.args.get('encoding')
if type == 'All types': if type == 'All types':
type = None type = None
if encoding == 'All encoding':
encoding = None
#date_from = '20180628' or date_from = '2018-06-28' #date_from = '20180628' or date_from = '2018-06-28'
#date_to = '20180628' or date_to = '2018-06-28' #date_to = '20180628' or date_to = '2018-06-28'
# verify file type input
if type is not None: if type is not None:
#retrieve + char #retrieve + char
type = type.replace(' ', '+') type = type.replace(' ', '+')
if type not in r_serv_metadata.smembers('hash_all_type'): if type not in r_serv_metadata.smembers('hash_all_type'):
type = None type = None
all_encoding = r_serv_metadata.smembers('all_decoder')
# verify encoding input
if encoding is not None:
if encoding not in all_encoding:
encoding = None
print(encoding)
date_range = [] date_range = []
if date_from is not None and date_to is not None: if date_from is not None and date_to is not None:
#change format #change format
@ -150,7 +163,10 @@ def hashDecoded_page():
l_64 = set() l_64 = set()
for date in date_range: for date in date_range:
if encoding is None:
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1) l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
else:
l_hash = r_serv_metadata.zrange(encoding+'_date:' +date, 0, -1)
if l_hash: if l_hash:
for hash in l_hash: for hash in l_hash:
l_64.add(hash) l_64.add(hash)
@ -199,7 +215,7 @@ def hashDecoded_page():
l_type = r_serv_metadata.smembers('hash_all_type') l_type = r_serv_metadata.smembers('hash_all_type')
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date, return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
date_from=date_from, date_to=date_to) encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to)
@hashDecoded.route('/hashDecoded/hash_by_type') @hashDecoded.route('/hashDecoded/hash_by_type')
def hash_by_type(): def hash_by_type():
@ -234,6 +250,14 @@ def showHash():
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen') last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
# get all encoding for this hash
list_hash_decoder = []
list_decoder = r_serv_metadata.smembers('all_decoder')
for decoder in list_decoder:
encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder')
if encoding is not None:
list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding})
num_day_type = 6 num_day_type = 6
date_range_sparkline = get_date_range(num_day_type) date_range_sparkline = get_date_range(num_day_type)
sparkline_values = list_sparkline_values(date_range_sparkline, hash) sparkline_values = list_sparkline_values(date_range_sparkline, hash)
@ -253,7 +277,7 @@ def showHash():
return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link, return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link,
b64_vt_report=b64_vt_report, b64_vt_report=b64_vt_report,
size=size, estimated_type=estimated_type, file_icon=file_icon, size=size, estimated_type=estimated_type, file_icon=file_icon,
first_seen=first_seen, first_seen=first_seen, list_hash_decoder=list_hash_decoder,
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values) last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
@app.route('/hashDecoded/downloadHash') @app.route('/hashDecoded/downloadHash')
@ -299,21 +323,67 @@ def hash_by_type_json():
type = type.replace(' ', '+') type = type.replace(' ', '+')
num_day_type = 30 num_day_type = 30
date_range_sparkline = get_date_range(num_day_type) date_range = get_date_range(num_day_type)
#verify input #verify input
if type in r_serv_metadata.smembers('hash_all_type'): if type in r_serv_metadata.smembers('hash_all_type'):
type_value = [] type_value = []
for date in date_range_sparkline: all_decoder = r_serv_metadata.smembers('all_decoder')
range_decoder = []
for date in date_range:
day_decoder = {}
day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for decoder in all_decoder:
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_decoder is None:
num_day_decoder = 0
day_decoder[decoder]= num_day_decoder
range_decoder.append(day_decoder)
return jsonify(range_decoder)
else:
return jsonify()
####################################
all_type = set()
for date in date_range:
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
if l_hash:
for hash in l_hash:
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
all_type.add(estimated_type)
range_type = []
for date in date_range:
if len(date_range) == 1:
if date==date_from and date==date_to:
for type in all_type:
day_type = {}
day_type['date']= type
list_decoder = r_serv_metadata.smembers('all_decoder')
for decoder in list_decoder:
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_decoder is None:
num_day_decoder = 0
day_type[decoder]= num_day_decoder
range_type.append(day_type)
else:
range_type = ''
else:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type in all_type:
num_day_type = r_serv_metadata.zscore('hash_type:'+type, date) num_day_type = r_serv_metadata.zscore('hash_type:'+type, date)
if num_day_type is None: if num_day_type is None:
num_day_type = 0 num_day_type = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] day_type[type]= num_day_type
type_value.append({ 'date' : date, 'value' : int( num_day_type )}) range_type.append(day_type)
return jsonify(type_value) return jsonify(range_type)
else:
return jsonify()
@hashDecoded.route('/hashDecoded/daily_type_json') @hashDecoded.route('/hashDecoded/daily_type_json')
def daily_type_json(): def daily_type_json():
@ -358,7 +428,23 @@ def range_type_json():
all_type.add(estimated_type) all_type.add(estimated_type)
range_type = [] range_type = []
for date in date_range: for date in date_range:
if len(date_range) == 1:
if date==date_from and date==date_to:
for type in all_type:
day_type = {}
day_type['date']= type
list_decoder = r_serv_metadata.smembers('all_decoder')
for decoder in list_decoder:
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
if num_day_decoder is None:
num_day_decoder = 0
day_type[decoder]= num_day_decoder
range_type.append(day_type)
else:
range_type = ''
else:
day_type = {} day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type in all_type: for type in all_type:

49
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -85,6 +85,17 @@
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span> <span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to"> <input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
</div> </div>
Encoding :
<select class="form-control" name="encoding" style="width=100%;">
<option>All encoding</option>
{% for encod in all_encoding %}
{% if encoding|string() == encod|string() %}
<option selected>{{ encod }}</option>
{% else %}
<option>{{ encod }}</option>
{% endif %}
{% endfor %}
</select>
File Type : File Type :
<select class="form-control" name="type" style="width=100%;"> <select class="form-control" name="type" style="width=100%;">
<option>All types</option> <option>All types</option>
@ -217,11 +228,11 @@
}); });
{% if type %} {% if type %}
chart.stackBarChart =barchart_type('/hashDecoded/hash_by_type_json?type={{type}}', '#barchart_type'); chart.stackBarChart =barchart_type_stack('/hashDecoded/hash_by_type_json?type={{type}}', 'id');
{% elif daily_type_chart %} {% elif daily_type_chart %}
chart.stackBarChart =barchart_type('/hashDecoded/daily_type_json?date={{daily_date}}', '#barchart_type'); chart.stackBarChart =barchart_type_stack('/hashDecoded/range_type_json?date_from={{daily_date}}&date_to={{daily_date}}', 'id');
{% else %} {% else %}
chart.stackBarChart = barchart_type_stack('url', 'id') chart.stackBarChart = barchart_type_stack("/hashDecoded/range_type_json?date_from={{date_from}}&date_to={{date_to}}", 'id')
{% endif %} {% endif %}
chart.onResize(); chart.onResize();
@ -294,7 +305,7 @@ function sparklines(id, points) {
</script> </script>
<script> <script>
var margin = {top: 20, right: 55, bottom: 30, left: 40}, var margin = {top: 20, right: 55, bottom: 55, left: 40},
width = 1000 - margin.left - margin.right, width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom; height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1); var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
@ -317,7 +328,7 @@ var svg = d3.select("#barchart_type").append("svg")
function barchart_type_stack(url, id) { function barchart_type_stack(url, id) {
d3.json("/hashDecoded/range_type_json?date_from={{date_from}}&date_to={{date_to}}") d3.json(url)
.then(function(data){ .then(function(data){
var labelVar = 'date'; //A var labelVar = 'date'; //A
@ -346,9 +357,17 @@ function barchart_type_stack(url, id) {
.call(xAxis) .call(xAxis)
.selectAll("text") .selectAll("text")
.attr("class", "bar") .attr("class", "bar")
.on("click", function (d) { window.location.href = "/hashDecoded/"+'?date_from='+d+'&date_to='+d }) {% if date_from|string == date_to|string and type is none %}
.style("text-anchor", "end") .on("click", function (d) { window.location.href = '/hashDecoded/?date_from={{date_from}}&date_to={{date_to}}&type='+d })
.attr("transform", "rotate(-45)" ); .attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
.attr("transform", "rotate(-18)" )
{% else %}
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
.attr("transform", "rotate(-40)" )
{% endif %}
.style("text-anchor", "end");
svg.append("g") svg.append("g")
.attr("class", "y axis") .attr("class", "y axis")
@ -376,7 +395,13 @@ function barchart_type_stack(url, id) {
.style("stroke", "grey") .style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); }) .on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); }) .on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and type is none %}
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?date_from={{date_from}}&date_to={{date_to}}&type='+d.label+'&encoding='+d.name; });
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type={{type}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
{% else %}
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; }); .on("click", function(d){ window.location.href = "/hashDecoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
{% endif %}
data.forEach(function(d) { data.forEach(function(d) {
if(d.total != 0){ if(d.total != 0){
@ -386,7 +411,13 @@ function barchart_type_stack(url, id) {
//.on("click", (window.location.href = "/hashDecoded/"+'?date_from='+d.date) ) //.on("click", (window.location.href = "/hashDecoded/"+'?date_from='+d.date) )
.attr('x', x(d.date) + x.bandwidth()/2) .attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total)) .attr('y', y(d.total))
{% if date_from|string == date_to|string and type is none %}
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from={{date_from}}&date_to={{date_to}}&type='+d.date })
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function () {window.location.href = '/hashDecoded/?type={{type}}&date_from='+d.date+'&date_to='+d.date })
{% else %}
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from='+d.date+'&date_to='+d.date }) .on("click", function () {window.location.href = "/hashDecoded/"+'?date_from='+d.date+'&date_to='+d.date })
{% endif %}
.style("text-anchor", "middle") .style("text-anchor", "middle")
.text(d.total); .text(d.total);
} }
@ -434,7 +465,7 @@ function showPopover (d) {
trigger: 'manual', trigger: 'manual',
html : true, html : true,
content: function() { content: function() {
return "date: " + d.label + return d.label +
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); } "<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
}); });
$(this).popover('show') $(this).popover('show')

16
var/www/modules/hashDecoded/templates/showHash.html
View file

@ -114,7 +114,7 @@
<th>First_seen</th> <th>First_seen</th>
<th>Last_seen</th> <th>Last_seen</th>
<th>Size (Kb)</th> <th>Size (Kb)</th>
<th>nb_seen_in_all_pastes</th> <th>Nb seen</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -172,6 +172,20 @@
</div> </div>
<div class="col-md-2"> <div class="col-md-2">
<div class="panel panel-info">
<div class="panel-heading">
<i class="fa fa-unlock-alt" aria-hidden="true"></i> Encoding
</div>
<div class="panel-body" style="text-align:center;">
{% for encoding in list_hash_decoder %}
<button id="" class="btn btn-default">
{{encoding['encoding']}} <span class="badge">{{encoding['nb_seen']}}</span>
</button>
{% endfor %}
</div>
</div>
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph <i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph