add: base 64 node graph

This commit is contained in:
Terrtia 2018-07-12 17:07:17 +02:00
parent fd02085495
commit 87b7facba8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 400 additions and 66 deletions

View file

@ -167,39 +167,39 @@ class Process(object):
return None
else:
#try:
if '.gz' in message:
path = message.split(".")[-2].split("/")[-1]
#find start of path with AIL_HOME
index_s = message.find(os.environ['AIL_HOME'])
#Stop when .gz
index_e = message.find(".gz")+3
if(index_s == -1):
complete_path = message[0:index_e]
try:
if '.gz' in message:
path = message.split(".")[-2].split("/")[-1]
#find start of path with AIL_HOME
index_s = message.find(os.environ['AIL_HOME'])
#Stop when .gz
index_e = message.find(".gz")+3
if(index_s == -1):
complete_path = message[0:index_e]
else:
complete_path = message[index_s:index_e]
else:
complete_path = message[index_s:index_e]
path = "-"
complete_path = "?"
else:
path = "-"
complete_path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
return message
curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
return message
#except:
#print('except')
#path = "?"
#value = str(timestamp) + ", " + path
#self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
#self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", "?")
#self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
#return message
except:
print('except')
path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", "?")
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
return message
def populate_set_out(self, msg, channel=None):
# multiproc

View file

@ -70,6 +70,22 @@ def get_file_icon(estimated_type):
return file_icon
def get_file_icon_text(estimated_type):
file_type = estimated_type.split('/')[0]
# set file icon
if file_type == 'application':
file_icon_text = '\uf15b'
elif file_type == 'audio':
file_icon_text = '\uf1c7'
elif file_type == 'image':
file_icon_text = '\uf03e'
elif file_type == 'text':
file_icon_text = '\uf15c'
else:
file_icon_text = '\uf15b'
return file_icon_text
def one():
return 1
@ -88,6 +104,9 @@ def base64Decoded_page():
date_to = request.args.get('date_to')
type = request.args.get('type')
if type == 'All types':
type = None
#date_from = '20180628' or date_from = '2018-06-28'
#date_to = '20180628' or date_to = '2018-06-28'
@ -189,10 +208,15 @@ def showHash():
hash = request.args.get('hash')
#hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea'
# TODO FIXME show error
if hash is None:
return base64Decoded_page()
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
# hash not found
# TODO FIXME show error
if estimated_type is None:
base64Decoded_page()
return base64Decoded_page()
else:
file_icon = get_file_icon(estimated_type)
@ -290,6 +314,66 @@ def range_type_json():
return jsonify(range_type)
@base64Decoded.route('/base64Decoded/hash_graph_node_json')
def hash_graph_node_json():
hash = request.args.get('hash')
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
if hash is not None and estimated_type is not None:
nodes_set_hash = set()
nodes_set_paste = set()
links_set = set()
url = hash
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
nodes_set_hash.add((hash, 1, first_seen, last_seen, estimated_type, nb_seen_in_paste, size, url))
#get related paste
l_pastes = r_serv_metadata.zrange('base64_hash:'+hash, 0, -1)
for paste in l_pastes:
url = paste
#nb_seen_in_this_paste = nb_in_file = int(r_serv_metadata.zscore('base64_hash:'+hash, paste))
nb_base64_in_paste = r_serv_metadata.scard('base64_paste:'+paste)
nodes_set_paste.add((paste, 2,nb_base64_in_paste,url))
links_set.add((hash, paste))
l_hash = r_serv_metadata.smembers('base64_paste:'+paste)
for child_hash in l_hash:
if child_hash != hash:
url = child_hash
first_seen = r_serv_metadata.hget('metadata_hash:'+child_hash, 'first_seen')
last_seen = r_serv_metadata.hget('metadata_hash:'+child_hash, 'last_seen')
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+child_hash, 'nb_seen_in_all_pastes')
size = r_serv_metadata.hget('metadata_hash:'+child_hash, 'size')
estimated_type = r_serv_metadata.hget('metadata_hash:'+child_hash, 'estimated_type')
nodes_set_hash.add((child_hash, 1, first_seen, last_seen, estimated_type, nb_seen_in_paste, size, url))
links_set.add((child_hash, paste))
#l_pastes_child = r_serv_metadata.zrange('base64_hash:'+child_hash, 0, -1)
#for child_paste in l_pastes_child:
nodes = []
for node in nodes_set_hash:
nodes.append({"id": node[0], "group": node[1], "first_seen": node[2], "last_seen": node[3], 'estimated_type': node[4], "nb_seen_in_paste": node[5], "size": node[6], 'icon': get_file_icon_text(node[4]),"url": url_for('base64Decoded.showHash', hash=node[7]), 'hash': True})
for node in nodes_set_paste:
nodes.append({"id": node[0], "group": node[1], "nb_seen_in_paste": node[2],"url": url_for('showsavedpastes.showsavedpaste', paste=node[3]), 'hash': False})
links = []
for link in links_set:
links.append({"source": link[0], "target": link[1]})
json = {"nodes": nodes, "links": links}
return jsonify(json)
else:
return jsonify({})
@base64Decoded.route('/base64Decoded/base64_types')
def base64_types():
date_from = 20180701

View file

@ -25,6 +25,9 @@
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.input-group .form-control {
position: unset;
}
.red_table thead{
background: #d91f2d;
color: #fff;
@ -84,7 +87,7 @@
</div>
File Type :
<select class="form-control" name="type" style="width=100%;">
<option></option>
<option>All types</option>
{% for typ in l_type %}
{% if type|string() == typ|string() %}
<option selected>{{ typ }}</option>
@ -95,7 +98,7 @@
</select>
<br>
<button class="btn btn-primary" style="text-align:center;">
<i class="fa fa-files-o"></i> Base64
<i class="fa fa-files-o"></i> Search
</button>
<form>
</div>
@ -159,7 +162,11 @@
</tbody>
</table>
{% else %}
<h3>{{ date_from }} to {{ date_to }}, No base64</h3>
{% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No base64</h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }}, No base64</h3>
{% endif %}
{% endif %}
</div>
@ -215,7 +222,6 @@
chart.stackBarChart =barchart_type('/base64Decoded/daily_type_json?date={{daily_date}}', '#barchart_type');
{% else %}
chart.stackBarChart = barchart_type_stack('url', 'id')
chart.onResize();
{% endif %}
chart.onResize();

View file

@ -25,31 +25,51 @@
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.red_table thead{
background: #d91f2d;
color: #fff;
}
.panelText {
color: black;
}
.line {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.bar {
fill: steelblue;
}
.bar:hover{
fill: brown;
cursor: pointer;
}
.bar_stack:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
line.link {
stroke: #666;
}
line.link:hover{
stroke: red;
stroke-width: 2px
}
.line_sparkline {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.node {
pointer-events: all;
}
circle {
stroke: none;
}
.graph_text_node {
font: 8px sans-serif;
pointer-events: none;
}
.node text {
font: 8px sans-serif;
pointer-events: auto;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
.graph_panel {
padding: unset;
}
</style>
</head>
<body>
@ -102,16 +122,52 @@
</li></ul>
</div></div>
<div class="row">
<div class="col-md-10">
<div class="panel panel-default">
<div class="panel-heading">
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
</div>
<div class="panel-body graph_panel">
<div id="graph">
</div>
</div>
</div>
</div>
<div class="col-md-2">
<div class="panel panel-default">
<div class="panel-heading">
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
</div>
<div class="panel-body" style="text-align:center;">
<button class="btn btn-primary" onclick="redraw_graph();">
<span class="glyphicon glyphicon-refresh"></span>&nbsp;Redraw Graph</div>
</button>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- /.row -->
<script>
var chart = {};
var all_graph = {};
$(document).ready(function(){
sparklines("sparkline", {{ sparkline_values }})
all_graph.node_graph = create_graph('/base64Decoded/hash_graph_node_json?hash={{hash}}');
all_graph.onResize();
});
$(window).on("resize", function() {
all_graph.onResize();
});
</script>
<script>
function updateVTReport(hash) {
@ -123,13 +179,20 @@
});
}
</script>
<script>
function redraw_graph() {
container_graph.attr("transform", "translate(40,0)")
.attr("transform", "scale(2)");
}
</script>
<script>
//var data = [6,3,3,2,5,3,9];
// a sparklines plot
function sparklines(id, points) {
var width = 100, height = 60;
var width_spark = 100, height_spark = 60;
var data = []
for (i = 0; i < points.length; i++) {
@ -140,11 +203,11 @@ function sparklines(id, points) {
}
var x = d3.scaleLinear()
.range([0, width - 10])
.range([0, width_spark - 10])
.domain([0,5]);
var y = d3.scaleLinear()
.range([height, 0])
.range([height_spark, 0])
.domain([0,10]);
var line = d3.line()
@ -152,10 +215,10 @@ function sparklines(id, points) {
.y(function(d) {return y(d.y)});
d3.select("#"+id).append('svg')
.attr('width', width)
.attr('height', height)
.attr('width', width_spark)
.attr('height', height_spark)
.append('path')
.attr('class','line')
.attr('class','line_sparkline')
.datum(data)
.attr('d', line);
@ -163,8 +226,189 @@ function sparklines(id, points) {
</script>
<script>
var width = 400,
height = 400;
var link;
var transform = d3.zoomIdentity;
var color = d3.scaleOrdinal(d3.schemeCategory10);
var div = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) { return d.id; }))
.force("charge", d3.forceManyBody())
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
.attr("height", height)
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
.on("dblclick.zoom", null)
var container_graph = svg_node.append("g")
.attr("transform", "translate(40,0)")
.attr("transform", "scale(2)");
function create_graph(url){
d3.json(url)
.then(function(data){
link = container_graph.append("g")
.selectAll("line")
.data(data.links)
.enter().append("line")
.attr("class", "link");
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
var node = container_graph.selectAll(".node")
.data(data.nodes)
.enter().append("g")
.attr("class", "nodes")
.on("dblclick", doubleclick)
.on("click", click)
.on("mouseover", mouseovered)
.on("mouseout", mouseouted)
.call(d3.drag()
.on("start", drag_start)
.on("drag", dragged)
.on("end", drag_end));
node.append("circle")
.attr("r", function(d) {
return (d.hash) ? 6 : 5; })
.attr("fill", function(d) {
if(!d.hash){return color(d.group)}
return 'white'; });
node.append('text')
.attr('text-anchor', 'middle')
.attr('dominant-baseline', 'central')
//.text(function(d) { return ICON_UNICODE[d.nodeType]; });
.attr('font-family', 'FontAwesome')
.attr('font-size', '12px' )
.text(function(d) {
if(d.hash){
return d.icon
} });
/* node.append("text")
.attr("dy", 3)
.attr("dx", 7)
.attr("class", "graph_text_node")
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.id; });*/
simulation
.nodes(data.nodes)
.on("tick", ticked);
simulation.force("link")
.links(data.links);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
/*node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });*/
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
}
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
}
function doubleclick (d) {
window.open(d.url, '_blank');
}
function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
// tooltip
var content;
if(d.hash == true){
content = "<b>"+d.id+"</b>"+"<br/>"+
"<br/>"+
"<i>First seen</i>: "+d.first_seen+"<br/>"+
"<i>Last seen</i>: "+d.last_seen+"<br/>"+
"<i>nb_seen_in_paste</i>: "+d.nb_seen_in_paste+"<br/>"+
"<i>Size (kb)</i>: "+d.size+"<br/>"+
"<br/>"+
"<i>Estimated type</i>: "+d.estimated_type;
} else {
content = "<b>"+d.id+"</b>"+"<br/>";
}
div.transition()
.duration(200)
.style("opacity", .9);
div.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
//links
/*link.style("stroke-opacity", function(o) {
return o.source === d || o.target === d ? 1 : opacity;
});*/
link.style("stroke", function(o){
return o.source === d || o.target === d ? "#666" : "#ddd";
});
}
function mouseouted() {
div.transition()
.duration(500)
.style("opacity", 0);
link.style("stroke", "#666");
}
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
</script>
</body>