add date_range stacked bar

This commit is contained in:
Terrtia 2018-07-06 16:54:43 +02:00
parent f7d38bea16
commit 0e5a7d8d47
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 445 additions and 13 deletions

View file

@ -75,10 +75,13 @@ def base64Decoded_page():
date_range = [] date_range = []
if date_from is not None and date_to is not None: if date_from is not None and date_to is not None:
#change format #change format
if len(date_from) != 8: try:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] if len(date_from) != 8:
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_range = substract_date(date_from, date_to) date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range: if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d")) date_range.append(datetime.date.today().strftime("%Y%m%d"))
@ -122,8 +125,6 @@ def base64Decoded_page():
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size') size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
if hash is not None and first_seen is not None and \ if hash is not None and first_seen is not None and \
last_seen is not None and \ last_seen is not None and \
nb_seen_in_paste is not None and \ nb_seen_in_paste is not None and \
@ -211,6 +212,52 @@ def daily_type_json():
return jsonify(type_value) return jsonify(type_value)
@base64Decoded.route('/base64Decoded/range_type_json')
def range_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
date_from = '20180601'
date_to = '20180706'
date_range = []
if date_from is not None and date_to is not None:
#change format
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
all_type = set()
for date in date_range:
l_hash = r_serv_metadata.zrange('base64_date:' +date, 0, -1)
if l_hash:
for hash in l_hash:
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
all_type.add(estimated_type)
range_type = []
for date in date_range:
day_type = {}
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type in all_type:
num_day_type = r_serv_metadata.zscore('base64_type:'+type, date)
if num_day_type is None:
num_day_type = 0
day_type[type]= num_day_type
range_type.append(day_type)
return jsonify(range_type)
@base64Decoded.route('/base64Decoded/base64_types')
def base64_types():
date_from = 20180701
date_to = 20180706
return render_template('base64_types.html', date_from=date_from, date_to=date_to)
@base64Decoded.route('/base64Decoded/send_file_to_vt', methods=['POST']) @base64Decoded.route('/base64Decoded/send_file_to_vt', methods=['POST'])
def send_file_to_vt(): def send_file_to_vt():
paste = request.form['paste'] paste = request.form['paste']

View file

@ -40,6 +40,9 @@
.bar:hover{ .bar:hover{
fill: brown; fill: brown;
} }
.svgText {
pointer-events: none;
}
</style> </style>
</head> </head>
<body> <body>
@ -58,14 +61,8 @@
<div class="row"> <div class="row">
<div class="col-md-10"> <div class="col-md-10">
{% if type %}
<div id="barchart_type"> <div id="barchart_type">
</div> </div>
{% endif %}
{% if daily_type_chart %}
<div id="barchart_type">
</div>
{% endif %}
</div> </div>
<div class="col-md-2"> <div class="col-md-2">
@ -209,7 +206,9 @@
{% if daily_type_chart %} {% if daily_type_chart %}
barchart_type('/base64Decoded/daily_type_json?date={{daily_date}}', '#barchart_type'); barchart_type('/base64Decoded/daily_type_json?date={{daily_date}}', '#barchart_type');
{% endif %} {% endif %}
{% if not daily_type_chart and not daily_type_chart%}
barchartstack_type('url', 'id');
{% endif %}
}); });
</script> </script>
<script> <script>
@ -372,6 +371,163 @@ function barchart_type(url, id) {
} }
</script> </script>
{% if not type and not daily_type_chart %}
<script>
var margin = {top: 20, right: 55, bottom: 30, left: 40},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type(url, id) {
/*var stack = d3.stack()
stack.values(function (d) { return d.values; })
.offset("zero")
.x(function (d) { return x(d.label); })
.y(function (d) { return d.value; });*/
/*var area = d3.svg.area()
.interpolate("cardinal")
.x(function (d) { return x(d.label); })
.y0(function (d) { return y(d.y0); })
.y1(function (d) { return y(d.y0 + d.y); });*/
//var color = d3.scale.ordinal().range(["#001c9c","#101b4d","#475003","#9c8305","#d3c47c"]);
d3.json("/base64Decoded/range_type_json")
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date).substring(5); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date).substring(5)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
.on("click", function(d){ window.location.href = "/base64Decoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 152)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 150)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: d.name,
placement: 'auto top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "date: " + d.label +
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
});
$(this).popover('show')
}
VIZ.onResize = function () {
var aspect = 1000 / 500, chart = $("#barchart_type");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.VIZ = VIZ;
</script>
{% endif %}
</body> </body>

View file

@ -0,0 +1,229 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.js') }}"></script>
<style>
.red_table thead{
background: #d91f2d;
color: #fff;
}
.bar_stack:hover{
fill: brown;
}
.bar_stack span {
position: relative;
}
.svgText {
pointer-events: none;
}
</style>
</head>
<body>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Base64 Files</h1>
</div>
<!-- /.col-lg-12 -->
<div id="chart">
</div>
</div>
<!-- /.row -->
<script>
var VIZ = {};
$(document).ready(function(){
activePage = "page-base64Decoded"
$("#"+activePage).addClass("active");
VIZ.stackBarChart = barchart_type('url', 'id')
VIZ.onResize();
});
$(window).on("resize", function() {
VIZ.onResize();
});
</script>
<script>
var margin = {top: 20, right: 55, bottom: 30, left: 40},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#chart").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type(url, id) {
/*var stack = d3.stack()
stack.values(function (d) { return d.values; })
.offset("zero")
.x(function (d) { return x(d.label); })
.y(function (d) { return d.value; });*/
/*var area = d3.svg.area()
.interpolate("cardinal")
.x(function (d) { return x(d.label); })
.y0(function (d) { return y(d.y0); })
.y1(function (d) { return y(d.y0 + d.y); });*/
//var color = d3.scale.ordinal().range(["#001c9c","#101b4d","#475003","#9c8305","#d3c47c"]);
d3.json("/base64Decoded/range_type_json")
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date).substring(5); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date).substring(5)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
.on("click", function(d){ window.location.href = "/base64Decoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 152)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 150)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: d.name,
placement: 'auto top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "date: " + d.label +
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
});
$(this).popover('show')
}
VIZ.onResize = function () {
var aspect = 1000 / 500, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.VIZ = VIZ;
</script>
</body>
</html>