From 7d442646c2c0d4d03358f0eade0d1ebdb25cb352 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 18 Nov 2019 16:35:40 +0100
Subject: [PATCH] chg: [Correlation UI] add metadata card and item link: paste
 + domain

---
 bin/lib/Correlate_object.py                   |   2 +-
 var/www/blueprints/correlation.py             |   8 +-
 .../showpaste/templates/show_saved_paste.html |   7 +
 .../correlation/metadata_card_domain.html     | 164 ++++++++++++++++++
 .../correlation/metadata_card_paste.html      | 146 ++++++++++++++++
 .../correlation/show_correlation.html         |   4 +
 .../crawler/crawler_splash/showDomain.html    |  10 ++
 7 files changed, 337 insertions(+), 4 deletions(-)
 create mode 100644 var/www/templates/correlation/metadata_card_domain.html
 create mode 100644 var/www/templates/correlation/metadata_card_paste.html

diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py
index afd82379..16b29d8f 100755
--- a/bin/lib/Correlate_object.py
+++ b/bin/lib/Correlate_object.py
@@ -38,7 +38,7 @@ def get_object_metadata(object_type, correlation_id, type_id=None):
     if object_type == 'domain':
         return Domain.Domain(correlation_id).get_domain_metadata()
     elif object_type == 'paste':
-        return {}
+        return Item.get_item({"id": correlation_id, "date": True, "tags": True})[0]
     elif object_type == 'decoded':
         return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True)
     elif object_type == 'pgp':
diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py
index ca2972de..819e6f3f 100644
--- a/var/www/blueprints/correlation.py
+++ b/var/www/blueprints/correlation.py
@@ -22,6 +22,7 @@ from Role_Manager import login_admin, login_analyst
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Correlate_object
+import Domain
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Cryptocurrency
@@ -108,9 +109,10 @@ def get_card_metadata(object_type, correlation_id, type_id=None):
         card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id)
         card_dict["vt"]["status"] = vt_enabled
     elif object_type == 'domain':
-        pass
+        card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
+        card_dict["tags"] = Domain.get_domain_tags(correlation_id)
     elif object_type == 'paste':
-        pass
+        card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
     return card_dict
 
 # ============= ROUTES ==============
@@ -180,7 +182,7 @@ def show_correlation():
         if type_id:
             dict_object["metadata"]['type_id'] = type_id
         dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id)
-        return render_template("show_correlation.html", dict_object=dict_object)
+        return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
 
 @correlation.route('/correlation/graph_node_json')
 @login_required
diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html
index 31d72436..623b9ea6 100644
--- a/var/www/modules/showpaste/templates/show_saved_paste.html
+++ b/var/www/modules/showpaste/templates/show_saved_paste.html
@@ -346,6 +346,13 @@
           {% endif %}
         </div>
 
+        <div>
+					<a href="{{ url_for('correlation.show_correlation')}}?object_type=paste&correlation_id={{ request.args.get('paste') }}&correlation_objects=paste" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fa fa-search"></i> Show Paste Correlation
+						</button>
+					</a>
+				</div>
+
       </div>
 
     <div class="panel-body" id="panel-body">
diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html
new file mode 100644
index 00000000..5cacd513
--- /dev/null
+++ b/var/www/templates/correlation/metadata_card_domain.html
@@ -0,0 +1,164 @@
+<div class="card my-3">
+	<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+		<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
+		<ul class="list-group mb-2">
+			<li class="list-group-item py-0">
+				<table class="table">
+					<thead>
+						<tr>
+							<th>Object type</th>
+							<th>type</th>
+							<th>First seen</th>
+							<th>Last check</th>
+							<th>Port</th>
+							<th>Status</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr>
+							<td>{{ dict_object["object_type"] }}</td>
+							<td>
+								<svg height="26" width="26">
+									<g class="nodes">
+										<circle cx="13" cy="13" r="13" fill="orange"></circle>
+										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+									</g>
+								</svg>
+								{{ dict_object["metadata"]["type_id"] }}
+							</td>
+							<td>{{ dict_object["metadata"]['first_seen'] }}</td>
+							<td>{{ dict_object["metadata"]['last_check'] }}</td>
+							<td>{{ dict_object["metadata"]['ports'] }}</td>
+							<td>
+								{% if dict_object["metadata"]["status"] %}
+									<div style="color:Green;">
+										<i class="fas fa-check-circle fa-2x"></i>
+										UP
+									</div>
+								{% else %}
+									<div style="color:Red;">
+										<i class="fas fa-times-circle fa-2x"></i>
+										DOWN
+									</div>
+								{% endif %}
+							</td>
+						</tr>
+					</tbody>
+				</table>
+			</li>
+
+			<li class="list-group-item py-0">
+				<br>
+				<div class="mb-3">
+					Tags:
+					{% for tag in dict_object["metadata_card"]['tags'] %}
+						<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+					{%  endfor %}
+				</div>
+				<div class="mb-2 float-right">
+					<a href="{{ url_for('crawler_splash.showDomain')}}?domain={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain
+						</button>
+					</a>
+				</div>
+
+			</li>
+		</ul>
+	</div>
+</div>
+
+<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+<script>
+	sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
+</script>
+
+<script>
+function create_line_chart(id, url){
+
+	var width = 900;
+	var height = Math.round(width / 4);
+
+	var margin = {top: 20, right: 55, bottom: 50, left: 40};
+
+	var x = d3.scaleTime().range([0, width]);
+	var y = d3.scaleLinear().rangeRound([height, 0]);
+
+	var xAxis = d3.axisBottom(x);
+	var yAxis = d3.axisLeft(y);
+
+	var parseTime = d3.timeParse("%Y-%m-%d");
+
+	var line = d3.line()
+							.x(function(d) {
+          			return x(d.date);
+        			}).y(function(d) {
+          			return y(d.value);
+        			});
+
+	var svg_line = d3.select('#'+id).append('svg')
+				.attr("id", "graph_div")
+				.attr("width", width + margin.left + margin.right)
+				.attr("height", height + margin.top + margin.bottom)
+				.append('g')
+						.attr('transform', "translate("+ margin.left +","+ margin.top +")");
+
+	var div = d3.select('body').append('div')
+				.attr('class', 'tooltip')
+				.style('opacity', 0);
+
+	//add div tooltip
+
+d3.json(url)
+	.then(function(data){
+
+		data.forEach(function(d) {
+			d.date_label = d.date;
+			d.date = parseTime(d.date);
+			d.value = +d.value;
+		});
+
+		// fit the data
+		x.domain(d3.extent(data, function(d) { return d.date; }));
+		//x.domain(data.map(function (d) { return d.date; })); //E
+		y.domain([0, d3.max(data, function(d){ return d.value ; })]);
+
+		//line
+		svg_line.append("path")
+				.data([data])
+				.attr("class", "line_graph")
+				.attr("d", line);
+
+		// add X axis
+		svg_line.append("g")
+		    .attr("transform", "translate(0," + height + ")")
+		    .call(d3.axisBottom(x))
+				.selectAll("text")
+					.style("text-anchor", "end")
+					.attr("transform", "rotate(-45)" );
+
+		// Add the Y Axis
+		svg_line.append("g")
+		    .call(d3.axisLeft(y));
+
+		//add a dot circle
+    svg_line.selectAll('dot')
+				.data(data).enter()
+				.append('circle')
+						.attr('r', 2)
+							.attr('cx', function(d) { return x(d.date); })
+							.attr('cy', function(d) { return y(d.value); })
+
+            	.on('mouseover', function(d) {
+	            	div.transition().style('opacity', .9);
+	            	div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
+								.style("left", (d3.event.pageX) + "px")
+								.style("top", (d3.event.pageY - 28) + "px");
+          		})
+            	.on('mouseout', function(d)
+          			{
+            			div.transition().style('opacity', 0);
+          			});
+
+	});
+}
+</script>
diff --git a/var/www/templates/correlation/metadata_card_paste.html b/var/www/templates/correlation/metadata_card_paste.html
new file mode 100644
index 00000000..6a9946ea
--- /dev/null
+++ b/var/www/templates/correlation/metadata_card_paste.html
@@ -0,0 +1,146 @@
+<div class="card my-3">
+	<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+		<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
+		<ul class="list-group mb-2">
+			<li class="list-group-item py-0">
+				<table class="table">
+					<thead>
+						<tr>
+							<th>Object type</th>
+							<th>type</th>
+							<th>date</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr>
+							<td>{{ dict_object["object_type"] }}</td>
+							<td>
+								<svg height="26" width="26">
+									<g class="nodes">
+										<circle cx="13" cy="13" r="13" fill="orange"></circle>
+										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+									</g>
+								</svg>
+								{{ dict_object["metadata"]["type_id"] }}
+							</td>
+							<td>{{ dict_object["metadata"]['date'][0:4] }}/{{ dict_object["metadata"]['date'][4:6] }}/{{ dict_object["metadata"]['date'][6:8] }}</td>
+						</tr>
+					</tbody>
+				</table>
+			</li>
+
+			<li class="list-group-item py-0">
+				<br>
+				<div class="mb-3">
+					Tags:
+					{% for tag in dict_object["metadata"]['tags'] %}
+						<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+					{%  endfor %}
+				</div>
+				<div class="mb-2 float-right">
+					<a href="{{ url_for('showsavedpastes.showsavedpaste')}}?paste={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fas fa-search"></i> Show Paste
+						</button>
+					</a>
+				</div>
+
+			</li>
+		</ul>
+	</div>
+</div>
+
+<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+<script>
+	sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
+</script>
+
+<script>
+function create_line_chart(id, url){
+
+	var width = 900;
+	var height = Math.round(width / 4);
+
+	var margin = {top: 20, right: 55, bottom: 50, left: 40};
+
+	var x = d3.scaleTime().range([0, width]);
+	var y = d3.scaleLinear().rangeRound([height, 0]);
+
+	var xAxis = d3.axisBottom(x);
+	var yAxis = d3.axisLeft(y);
+
+	var parseTime = d3.timeParse("%Y-%m-%d");
+
+	var line = d3.line()
+							.x(function(d) {
+          			return x(d.date);
+        			}).y(function(d) {
+          			return y(d.value);
+        			});
+
+	var svg_line = d3.select('#'+id).append('svg')
+				.attr("id", "graph_div")
+				.attr("width", width + margin.left + margin.right)
+				.attr("height", height + margin.top + margin.bottom)
+				.append('g')
+						.attr('transform', "translate("+ margin.left +","+ margin.top +")");
+
+	var div = d3.select('body').append('div')
+				.attr('class', 'tooltip')
+				.style('opacity', 0);
+
+	//add div tooltip
+
+d3.json(url)
+	.then(function(data){
+
+		data.forEach(function(d) {
+			d.date_label = d.date;
+			d.date = parseTime(d.date);
+			d.value = +d.value;
+		});
+
+		// fit the data
+		x.domain(d3.extent(data, function(d) { return d.date; }));
+		//x.domain(data.map(function (d) { return d.date; })); //E
+		y.domain([0, d3.max(data, function(d){ return d.value ; })]);
+
+		//line
+		svg_line.append("path")
+				.data([data])
+				.attr("class", "line_graph")
+				.attr("d", line);
+
+		// add X axis
+		svg_line.append("g")
+		    .attr("transform", "translate(0," + height + ")")
+		    .call(d3.axisBottom(x))
+				.selectAll("text")
+					.style("text-anchor", "end")
+					.attr("transform", "rotate(-45)" );
+
+		// Add the Y Axis
+		svg_line.append("g")
+		    .call(d3.axisLeft(y));
+
+		//add a dot circle
+    svg_line.selectAll('dot')
+				.data(data).enter()
+				.append('circle')
+						.attr('r', 2)
+							.attr('cx', function(d) { return x(d.date); })
+							.attr('cy', function(d) { return y(d.value); })
+
+            	.on('mouseover', function(d) {
+	            	div.transition().style('opacity', .9);
+	            	div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
+								.style("left", (d3.event.pageX) + "px")
+								.style("top", (d3.event.pageY - 28) + "px");
+          		})
+            	.on('mouseout', function(d)
+          			{
+            			div.transition().style('opacity', 0);
+          			});
+
+	});
+}
+</script>
diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html
index f6e00ae3..bcf4d19a 100644
--- a/var/www/templates/correlation/show_correlation.html
+++ b/var/www/templates/correlation/show_correlation.html
@@ -94,6 +94,10 @@
 						{% include 'correlation/metadata_card_pgp.html' %}
 					{% elif dict_object["object_type"] == "decoded" %}
 						{% include 'correlation/metadata_card_decoded.html' %}
+					{% elif dict_object["object_type"] == "domain" %}
+						{% include 'correlation/metadata_card_domain.html' %}
+					{% elif dict_object["object_type"] == "paste" %}
+						{% include 'correlation/metadata_card_paste.html' %}
 					{% endif %}
 
 					<div class="row">
diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html
index 16161fe9..75bc88b6 100644
--- a/var/www/templates/crawler/crawler_splash/showDomain.html
+++ b/var/www/templates/crawler/crawler_splash/showDomain.html
@@ -98,6 +98,16 @@
 								Last Origin: <a class="badge" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=dict_domain['origin_item']) }}" />{{ dict_domain['origin_item'] }}</a>
 								{%endif%}
 
+                {% if 'decoded' in dict_domain or 'pgp' in dict_domain or 'cryptocurrency' in dict_domain%}
+                  <hr>
+                  <div class="mt-2">
+          					<a href="{{ url_for('correlation.show_correlation')}}?object_type=domain&correlation_id={{ dict_domain['domain'] }}&correlation_objects=domain" target="_blank" style="font-size: 15px">
+          						<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain Correlations
+          						</button>
+          					</a>
+          				</div>
+                {%endif%}
+
 							</div>
 						</div>