chg: [UI-Hash] add decoder pie

OVERVIEW.md

@@ -59,3 +59,7 @@ ARDB overview
 	ZADD - 'hash_type:'+type	date			nb_seen
 	ZADD - 'base64_type:'+type	date			nb_seen
 	ZADD - 'binary_type:'+type	date			nb_seen
+
+	GET  - 'base64_decoded:'+date	nd_decoded
+	GET  - 'binary_decoded:'+date	nd_decoded
+

bin/Decoder.py

@@ -61,7 +61,6 @@ def decode_string(content, message, date, encoded_list, decoder_name, encoded_mi
 def save_hash(decoder_name, message, date, decoded):
     print(decoder_name)
     type = magic.from_buffer(decoded, mime=True)
-    print(type)
     hash = sha1(decoded).hexdigest()
     print(hash)
 
@@ -75,6 +74,7 @@ def save_hash(decoder_name, message, date, decoded):
     date_paste = '{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])
     date_key = date[0:4] + date[4:6] + date[6:8]
 
+    serv_metadata.incrby(decoder_name+'_decoded:'+date_key, 1)
     serv_metadata.zincrby('hash_date:'+date_key, hash, 1)
     serv_metadata.zincrby(decoder_name+'_date:'+date_key, hash, 1)
 

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -100,7 +100,6 @@ def all_hash_search():
     date_to = request.form.get('date_to')
     type = request.form.get('type')
     encoding = request.form.get('encoding')
-    print(encoding)
     return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding))
 
 @hashDecoded.route("/hashDecoded/", methods=['GET'])
@@ -131,7 +130,6 @@ def hashDecoded_page():
     if encoding is not None:
         if encoding not in all_encoding:
             encoding = None
-    print(encoding)
 
     date_range = []
     if date_from is not None and date_to is not None:
@@ -347,43 +345,45 @@ def hash_by_type_json():
     else:
         return jsonify()
 
-    ####################################
+@hashDecoded.route('/hashDecoded/decoder_type_json')
-    all_type = set()
+def decoder_type_json():
-    for date in date_range:
+    date_from = request.args.get('date_from')
-        l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
+    date_to = request.args.get('date_to')
-        if l_hash:
-            for hash in l_hash:
-                estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
-                all_type.add(estimated_type)
 
-    range_type = []
+    type = request.args.get('type')
+    encoding = request.args.get('encoding')
 
+    all_decoder = r_serv_metadata.smembers('all_decoder')
+
+    date_range = []
+    if date_from is not None and date_to is not None:
+        #change format
+        try:
+            if len(date_from) != 8:
+                date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
+                date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
+            date_range = substract_date(date_from, date_to)
+        except:
+            pass
+
+    if not date_range:
+        date_range.append(datetime.date.today().strftime("%Y%m%d"))
+
+    nb_decoded = {}
     for date in date_range:
-        if len(date_range) == 1:
+        for decoder in all_decoder:
-            if date==date_from and date==date_to:
+            if type is None:
-                for type in all_type:
+                nb_decoded[decoder] = r_serv_metadata.get(decoder+'_decoded:'+date)
-                    day_type = {}
-                    day_type['date']= type
-                    list_decoder = r_serv_metadata.smembers('all_decoder')
-                    for decoder in list_decoder:
-                        num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
-                        if num_day_decoder is None:
-                            num_day_decoder = 0
-                        day_type[decoder]= num_day_decoder
-                    range_type.append(day_type)
             else:
-                range_type = ''
+                nb_decoded[decoder] = r_serv_metadata.hget(decoder+'_type:'+type, date)
-        else:
+            if nb_decoded[decoder] is None:
-            day_type = {}
+                nb_decoded[decoder] = 0
-            day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
+
-            for type in all_type:
+    to_json = []
-                num_day_type = r_serv_metadata.zscore('hash_type:'+type, date)
+    for decoder in all_decoder:
-                if num_day_type is None:
+        to_json.append({'name': decoder, 'value': nb_decoded[decoder]})
-                    num_day_type = 0
+    return jsonify(to_json)
-                day_type[type]= num_day_type
-            range_type.append(day_type)
 
-    return jsonify(range_type)
 
 @hashDecoded.route('/hashDecoded/daily_type_json')
 def daily_type_json():

var/www/modules/hashDecoded/templates/hashDecoded.html

@@ -50,6 +50,17 @@
 			.svgText {
     		pointer-events: none;
 			}
+			div.tooltip {
+				position: absolute;
+				text-align: center;
+				padding: 2px;
+				font: 12px sans-serif;
+				background: #ebf4fb;
+				border: 2px solid #b7ddf2;
+				border-radius: 8px;
+				pointer-events: none;
+				color: #000000;
+			}
 		</style>
 	</head>
 	<body>
@@ -114,6 +125,8 @@
 										<form>
 									</div>
 								</div>
+								<div id="pie_chart_encoded">
+			          </div>
 							</div>
 						</div>
 					</div>
@@ -482,7 +495,78 @@ window.chart = chart;
 
 </script>
 
+<script>
 
+var width_pie = 200;
+var height_pie = 200;
+var padding_pie = 10;
+var opacity_pie = .8;
+
+var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
+//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
+var color_pie = d3.scaleOrdinal(d3.schemeSet3);
+
+var div_pie = d3.select("body").append("div")
+		.attr("class", "tooltip")
+		.style("opacity", 0);
+
+var svg_pie = d3.select("#pie_chart_encoded")
+		.append('svg')
+		.attr("width", '100%')
+		.attr("height", '100%')
+		.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
+		.attr('preserveAspectRatio','xMinYMin')
+
+
+var g_pie = svg_pie.append('g')
+		.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
+
+var arc_pie = d3.arc()
+		.innerRadius(0)
+		.outerRadius(radius_pie);
+
+d3.json("/hashDecoded/decoder_type_json?date_from={{date_from}}&date_to={{date_to}}")
+	.then(function(data){
+
+	var pie_pie = d3.pie()
+			.value(function(d) { return d.value; });
+
+	var path_pie = g_pie.selectAll('path')
+		  .data(pie_pie(data))
+		  .enter()
+		  .append("g")
+		  .append('path')
+		  .attr('d', arc_pie)
+		  .attr('fill', (d,i) => color_pie(i))
+			.on("mouseover", mouseovered_pie)
+			.on("mouseout", mouseouted_pie)
+		  .style('opacity', opacity_pie)
+		  .style('stroke', 'white');
+	});
+
+function mouseovered_pie(d) {
+
+	// tooltip
+	var content;
+
+	content = "<b>"+d.data.name+"</b>"+"<br/>"+
+						"<br/>"+
+						"<i>Decoded</i>: "+d.data.value+"<br/>"
+
+	div_pie.transition()
+		.duration(200)
+		.style("opacity", .9);
+	div_pie.html(content)
+		.style("left", (d3.event.pageX) + "px")
+		.style("top", (d3.event.pageY - 28) + "px");
+}
+
+function mouseouted_pie() {
+	div_pie.transition()
+		.duration(500)
+		.style("opacity", 0);
+}
+</script>