chg: [relationships] add relationship engine + WIP relationships between forwarded messages/chats

bin/importer/FeederImporter.py

@@ -100,6 +100,7 @@ class FeederImporter(AbstractImporter):
         else:
             objs = set()
 
+        if data_obj:
             objs.add(data_obj)
 
         for obj in objs:
@@ -107,7 +108,7 @@ class FeederImporter(AbstractImporter):
                 gzip64_content = feeder.get_gzip64_content()
                 return obj, f'{feeder_name} {gzip64_content}'
             else:  # Messages save on DB
-                if obj.exists():
+                if obj.exists() and obj.type != 'chat':
                     return obj, f'{feeder_name}'
 
 
@@ -136,4 +137,5 @@ class FeederModuleImporter(AbstractModule):
 # Launch Importer
 if __name__ == '__main__':
     module = FeederModuleImporter()
+    # module.debug = True
     module.run()

bin/importer/feeders/abstract_chats_feeder.py

@@ -93,7 +93,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         return self.json_data['meta'].get('reactions', [])
 
     def get_message_timestamp(self):
-        return self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
+        if not self.json_data['meta'].get('date'):
+            return None
+        else:
+            return self.json_data['meta']['date']['timestamp']
         # if self.json_data['meta'].get('date'):
         #     date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
         #     date = date.strftime('%Y/%m/%d')
@@ -115,17 +118,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
     def get_message_reply_id(self):
         return self.json_data['meta'].get('reply_to', {}).get('message_id')
 
+    def get_message_forward(self):
+        return self.json_data['meta'].get('forward')
+
     def get_message_content(self):
         decoded = base64.standard_b64decode(self.json_data['data'])
         return _gunzip_bytes_obj(decoded)
 
-    def get_obj(self):  # TODO handle others objects -> images, pdf, ...
+    def get_obj(self):
         #### TIMESTAMP ####
         timestamp = self.get_message_timestamp()
 
         #### Create Object ID ####
         chat_id = self.get_chat_id()
+        try:
             message_id = self.get_message_id()
+        except KeyError:
+            if chat_id:
+                self.obj = Chat(chat_id, self.get_chat_instance_uuid())
+                return self.obj
+            else:
+                self.obj = None
+                return None
+
         thread_id = self.get_thread_id()
         # channel id
         # thread id
@@ -236,7 +251,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         #   # ADD NEW MESSAGE REF (used by discord)
 
     def process_sender(self, new_objs, obj, date, timestamp):
-        meta = self.json_data['meta']['sender']
+        meta = self.json_data['meta'].get('sender')
+        if not meta:
+            return None
+
         user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
 
         # date stat + correlation
@@ -286,8 +304,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         # REPLY
         reply_id = self.get_message_reply_id()
 
-        # TODO Translation
-
         print(self.obj.type)
 
         # TODO FILES + FILES REF
@@ -295,7 +311,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         # get object by meta object type
         if self.obj.type == 'message':
             # Content
-            obj = Messages.create(self.obj.id, self.get_message_content())  # TODO translation
+            obj = Messages.create(self.obj.id, self.get_message_content())
 
             # FILENAME
             media_name = self.get_media_name()
@@ -305,7 +321,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
 
             for reaction in self.get_reactions():
                 obj.add_reaction(reaction['reaction'], int(reaction['count']))
-
+        elif self.obj.type == 'chat':
+            pass
         else:
             chat_id = self.get_chat_id()
             thread_id = self.get_thread_id()
@@ -341,9 +358,26 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             # CHAT
             chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
+            # Message forward
+            # if self.get_json_meta().get('forward'):
+            #     forward_from = self.get_message_forward()
+            #     print('-----------------------------------------------------------')
+            #     print(forward_from)
+            #     if forward_from:
+            #         forward_from_type = forward_from['from']['type']
+            #         if forward_from_type == 'channel' or forward_from_type == 'chat':
+            #             chat_forward_id = forward_from['from']['id']
+            #             chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
+            #             if chat_forward.exists():
+            #                 for chat_obj in chat_objs:
+            #                     if chat_obj.type == 'chat':
+            #                         chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
+            #                         # chat_forward.add_relationship(obj.get_global_id(), 'forward')
+
             # SENDER # TODO HANDLE NULL SENDER
             user_account = self.process_sender(new_objs, obj, date, timestamp)
 
+            if user_account:
             # UserAccount---ChatObjects
                 for obj_chat in chat_objs:
                     user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)

bin/lib/objects/Chats.py

@@ -76,6 +76,7 @@ class Chat(AbstractChatObject):
         meta['tags'] = self.get_tags(r_list=True)
         if 'icon' in options:
             meta['icon'] = self.get_icon()
+            meta['img'] = meta['icon']
         if 'info' in options:
             meta['info'] = self.get_info()
         if 'participants' in options:
@@ -93,6 +94,8 @@ class Chat(AbstractChatObject):
         if 'threads' in options:
             meta['threads'] = self.get_threads()
             print(meta['threads'])
+        if 'tags_safe' in options:
+            meta['tags_safe'] = self.is_tags_safe(meta['tags'])
         return meta
 
     def get_misp_object(self):

bin/lib/objects/Messages.py

@@ -148,15 +148,32 @@ class Message(AbstractObject):
     def add_reaction(self, reactions, nb_reaction):
         r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
 
-    # Update value on import
+    # Interactions between users -> use replies
-    # reply to -> parent ?
-    # reply/comment - > children ?
     # nb views
-    # reactions
+    # MENTIONS -> Messages + Chats
-    # nb fowards
+    #       # relationship -> mention       - Chat    -> Chat
-    # room ???
+    #                                       - Message -> Chat
-    # message from channel ???
+    #                                       - Message -> Message ??? fetch mentioned messages
+    # FORWARDS
+    # TODO Create forward CHAT -> message
+    #                       message (is forwarded) -> message (is forwarded from) ???
+    #                       # TODO get source message timestamp
+    #
+    #       # is forwarded
+    #       # forwarded from -> check if relationship
+    #       # nb forwarded -> scard relationship
+    #
+    #       Messages -> CHATS -> NB forwarded
+    #       CHAT -> NB forwarded by chats -> NB messages -> parse full set ????
+    #
+    #
+    #
+    #
+    #
+    #
+    # show users chats
     # message media
+    # flag is deleted -> event or missing from feeder pass ???
 
     def get_translation(self, content=None, source=None, target='fr'):
         """

bin/lib/objects/abstract_object.py

@@ -24,6 +24,7 @@ from lib.ConfigLoader import ConfigLoader
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
+from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -284,6 +285,22 @@ class AbstractObject(ABC):
 
     ## -Correlation- ##
 
+    ## Relationship ##
+
+    def get_nb_relationships(self, filter=[]):
+        return get_obj_nb_relationships(self.get_global_id())
+
+    def add_relationship(self, obj2_global_id, relationship, source=True):
+        # is source
+        if source:
+            print(self.get_global_id(), obj2_global_id, relationship)
+            add_obj_relationship(self.get_global_id(), obj2_global_id, relationship)
+        # is target
+        else:
+            add_obj_relationship(obj2_global_id, self.get_global_id(), relationship)
+
+    ## -Relationship- ##
+
     ## Parent ##
 
     def is_parent(self):

bin/lib/objects/ail_objects.py

@@ -10,6 +10,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib.ConfigLoader import ConfigLoader
 from lib.ail_core import get_all_objects, get_object_all_subtypes
 from lib import correlations_engine
+from lib import relationships_engine
 from lib import btc_ail
 from lib import Tag
 
@@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
 
 # --- CORRELATION --- #
 
+def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
+    obj = get_object(obj_type, subtype, obj_id)
+    return obj.get_nb_relationships(filter=filter_types)
+
+def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+                                 objs_hidden=set(),
+                                 flask_context=False):
+    obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
+                                                                    filter_types=filter_types,
+                                                                    max_nodes=max_nodes, level=level,
+                                                                    objs_hidden=objs_hidden)
+    # print(meta)
+    meta['objs'] = list(meta['objs'])
+    return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
+            "links": links,
+            "meta": meta}
+
 
 # if __name__ == '__main__':
     # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

bin/lib/relationships_engine.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+
+config_loader = ConfigLoader()
+r_rel = config_loader.get_db_conn("Kvrocks_Relationships")
+config_loader = None
+
+
+RELATIONSHIPS = {
+    "forward",
+    "mention"
+}
+def get_relationships():
+    return RELATIONSHIPS
+
+
+def get_obj_relationships_by_type(obj_global_id, relationship):
+    return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship):
+    return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_relationships(obj_global_id):
+    relationships = []
+    for relationship in get_relationships():
+        for rel in get_obj_relationships_by_type(obj_global_id, relationship):
+            meta = {'relationship': relationship}
+            direction, obj_id = rel.split(':', 1)
+            if direction == 'i':
+                meta['source'] = obj_id
+                meta['target'] = obj_global_id
+            else:
+                meta['target'] = obj_id
+                meta['source'] = obj_global_id
+
+            if not obj_id.startswith('chat'):
+                continue
+
+            meta['id'] = obj_id
+            # meta['direction'] = direction
+            relationships.append(meta)
+    return relationships
+
+def get_obj_nb_relationships(obj_global_id):
+    nb = {}
+    for relationship in get_relationships():
+        nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
+    return nb
+
+
+# TODO Filter by obj type ???
+def add_obj_relationship(source, target, relationship):
+    r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
+    r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
+    # r_rel.sadd(f'rels:{source}', relationship)
+    # r_rel.sadd(f'rels:{target}', relationship)
+
+
+def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+    links = []
+    nodes = set()
+    meta = {'complete': True, 'objs': set()}
+    done = set()
+    done_link = set()
+
+    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+    return nodes, links, meta
+
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+    meta['objs'].add(obj_global_id)
+    nodes.add(obj_global_id)
+
+    for rel in get_obj_relationships(obj_global_id):
+        meta['objs'].add(rel['id'])
+
+        if rel['id'] in done:
+            continue
+
+        if len(nodes) > max_nodes != 0:
+            meta['complete'] = False
+            break
+
+        nodes.add(rel['id'])
+
+        str_link = f"{rel['source']}{rel['target']}{rel['relationship']}"
+        if str_link not in done_link:
+            links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']})
+            done_link.add(str_link)
+
+        if level > 0:
+            next_level = level - 1
+
+            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+
+    # done.add(rel['id'])
+
+
+if __name__ == '__main__':
+    source = ''
+    target = ''
+    add_obj_relationship(source, target, 'forward')
+    # print(get_obj_relationships(source))

configs/6383.conf

@@ -664,6 +664,7 @@ namespace.db ail_datas
 namespace.dup ail_dups
 namespace.obj ail_objs
 namespace.tl ail_tls
+namespace.rel ail_rels
 namespace.stat ail_stats
 namespace.tag ail_tags
 namespace.track ail_trackers

configs/core.cfg.sample

@@ -196,6 +196,11 @@ host = localhost
 port = 6383
 password = ail_objs
 
+[Kvrocks_Relationships]
+host = localhost
+port = 6383
+password = ail_rels
+
 [Kvrocks_Timeline]
 host = localhost
 port = 6383

var/www/blueprints/correlation.py

@@ -203,7 +203,7 @@ def get_description():
         return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
     # object exist
     else:
-        res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'tags_safe'},
+        res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'},
                                           flask_context=True)
         if 'tags' in res:
             res['tags'] = list(res['tags'])
@@ -292,3 +292,64 @@ def correlation_tags_add():
                             max_nodes=nb_max,
                             hidden=hidden, hidden_str=",".join(hidden),
                             filter=",".join(filter_types)))
+
+#####################################################################################
+
+@correlation.route('/relationships/graph_node_json')
+@login_required
+@login_read_only
+def relationships_graph_node_json():
+    obj_id = request.args.get('id')
+    subtype = request.args.get('subtype')
+    obj_type = request.args.get('type')
+    max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
+    level = sanitise_level(request.args.get('level'))
+
+    json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
+    return jsonify(json_graph)
+
+
+@correlation.route('/relationship/show', methods=['GET', 'POST'])
+@login_required
+@login_read_only
+def show_relationship():
+    if request.method == 'POST':
+        object_type = request.form.get('obj_type')
+        subtype = request.form.get('subtype')
+        obj_id = request.form.get('obj_id')
+        max_nodes = request.form.get('max_nb_nodes_in')
+        level = sanitise_level(request.form.get('level'))
+
+        # redirect to keep history and bookmark
+        return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
+                                max_nodes=max_nodes, level=level))
+
+    # request.method == 'GET'
+    else:
+        obj_type = request.args.get('type')
+        subtype = request.args.get('subtype', '')
+        obj_id = request.args.get('id')
+        max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
+        level = sanitise_level(request.args.get('level'))
+
+        # check if obj_id exist
+        if not ail_objects.exists_obj(obj_type, subtype, obj_id):
+            return abort(404)
+        # object exist
+        else: # TODO remove old dict key
+            dict_object = {"type": obj_type,
+                           "id": obj_id,
+                           "object_type": obj_type,
+                           "max_nodes": max_nodes, "level": level,
+                           "correlation_id": obj_id,
+                           "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
+                           "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
+                           }
+            if subtype:
+                dict_object["subtype"] = subtype
+                dict_object["metadata"]['type_id'] = subtype
+            else:
+                dict_object["subtype"] = ''
+                dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
+            return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
+                                       tags_selector_data=Tag.get_tags_selector_data())

var/www/templates/correlation/metadata_card_chat.html

@@ -0,0 +1,78 @@
+<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#}
+{#	{% include 'modals/add_tags.html' %}#}
+{#{% endwith %}#}
+
+{% include 'modals/edit_tag.html' %}
+
+<div class="card my-3">
+	<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+        <h4>{{ dict_object["correlation_id"] }}</h4>
+        {{ dict_object }}
+        <div class="text-secondary">{{ dict_object["correlation_id"] }}</div>
+		<ul class="list-group mb-2">
+			<li class="list-group-item py-0">
+				<div class="row">
+					<div class="col-md-10">
+						<table class="table">
+							<thead>
+								<tr>
+									<th>Object type</th>
+									<th>First seen</th>
+									<th>Last seen</th>
+									<th>Nb seen</th>
+								</tr>
+							</thead>
+							<tbody>
+								<tr>
+									<td>
+										<svg height="26" width="26">
+											<g class="nodes">
+												<circle cx="13" cy="13" r="13" fill="orange"></circle>
+												<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata"]["icon"]["icon"] }}</text>
+											</g>
+										</svg>
+                                        {{ dict_object["object_type"] }}
+									</td>
+									<td>{{ dict_object["metadata"]['first_seen'] }}</td>
+									<td>{{ dict_object["metadata"]['last_seen'] }}</td>
+									<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
+								</tr>
+							</tbody>
+						</table>
+					</div>
+					<div class="col-md-1">
+						<div id="sparkline"></div>
+					</div>
+				</div>
+			</li>
+
+            <li class="list-group-item py-0">
+				<br>
+				<div class="mb-3">
+					Tags:
+					{% for tag in dict_object["metadata"]['tags'] %}
+						<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
+						data-tagid="{{ tag }}" data-objtype="cookie-name" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
+							{{ tag }}
+						</button>
+					{%  endfor %}
+					<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+          	            <i class="far fa-plus-square"></i>
+                    </button>
+				</div>
+			</li>
+		</ul>
+
+		{% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
+	</div>
+</div>
+

var/www/templates/correlation/show_correlation.html

@@ -541,6 +541,15 @@ d3.json(url)
 					.on("drag", dragged)
 					.on("end", drag_end));
 
+    /*
+    node.append("image")
+    .attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png")
+    .attr("height", 20)
+    .attr("width", 20)
+    .attr("x", -10)
+    .attr("y", -10);
+
+     */
 
 	node.append("circle")
 			.attr("r", function(d) {

var/www/templates/correlation/show_relationship.html

@@ -0,0 +1,719 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+
+	  <title>AIL - framework</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+	  <!-- Core CSS -->
+		<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+		<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	  <!-- JS -->
+		<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+
+		<style>
+		.icon_legend {
+			color: #0c5460;
+			background-color: #bee5eb;
+		}
+
+		line.link {
+			stroke: #666;
+		}
+		line.link:hover{
+			stroke: red;
+			stroke-width: 2px;
+		}
+		.node {
+			pointer-events: all;
+		}
+
+		circle {
+			stroke: none;
+		}
+
+		.graph_text_node {
+			font: 8px sans-serif;
+			pointer-events: none;
+		}
+
+		.graph_node_icon {
+			pointer-events: none;
+		}
+
+		.node text {
+			font: 8px sans-serif;
+			pointer-events: auto;
+		}
+
+		div.tooltip_graph {
+			position: absolute;
+			text-align: center;
+			padding: 2px;
+			font: 12px sans-serif;
+			background: #ebf4fb;
+			border: 2px solid #b7ddf2;
+			border-radius: 8px;
+			pointer-events: none;
+			color: #000000;
+		}
+
+        .pixelated {
+            image-rendering: pixelated;
+        }
+
+        .blured {
+            filter: blur(5px);
+        }
+
+		.graph_panel {
+			padding: unset;
+		}
+
+		.line_graph {
+		  fill: none;
+		  stroke: steelblue;
+		  stroke-width: 2px;
+			stroke-linejoin: round;
+			stroke-linecap: round;
+			stroke-width: 1.5;
+			/*attr('stroke', '#bcbd22').*/
+		}
+		</style>
+	</head>
+	<body>
+
+		{% include 'nav_bar.html' %}
+
+		<div class="container-fluid">
+			<div class="row">
+
+				{% include 'sidebars/sidebar_objects.html' %}
+
+				<div class="col-12 col-lg-10" id="core_content">
+
+					{% if dict_object["object_type"] == "pgp" %}
+						{% include 'correlation/metadata_card_pgp.html' %}
+					{% elif dict_object["object_type"] == "cryptocurrency" %}
+						{% include 'correlation/metadata_card_cryptocurrency.html' %}
+					{% elif dict_object["object_type"] == "username" %}
+							{% include 'correlation/metadata_card_username.html' %}
+					{% elif dict_object["object_type"] == "decoded" %}
+						{% include 'correlation/metadata_card_decoded.html' %}
+                    {% elif dict_object["object_type"] == "chat" %}
+						{% include 'correlation/metadata_card_chat.html' %}
+                    {% elif dict_object["object_type"] == "cve" %}
+						{% include 'correlation/metadata_card_cve.html' %}
+					{% elif dict_object["object_type"] == "domain" %}
+						{% include 'correlation/metadata_card_domain.html' %}
+					{% elif dict_object["object_type"] == "screenshot" %}
+						{% include 'correlation/metadata_card_screenshot.html' %}
+                    {% elif dict_object["object_type"] == "title" %}
+						{% include 'correlation/metadata_card_title.html' %}
+                    {% elif dict_object["object_type"] == "cookie-name" %}
+						{% include 'correlation/metadata_card_cookie_name.html' %}
+                    {% elif dict_object["object_type"] == "etag" %}
+						{% include 'correlation/metadata_card_etag.html' %}
+                    {% elif dict_object["object_type"] == "hhhash" %}
+						{% include 'correlation/metadata_card_hhhash.html' %}
+					{% elif dict_object["object_type"] == "item" %}
+						{% include 'correlation/metadata_card_item.html' %}
+					{% endif %}
+
+					<div class="row">
+						<div class="col-xl-10">
+
+							<div class="card mb-3">
+								<div class="card-header">
+									<i class="fas fa-project-diagram"></i> Graph
+									<span class="float-right">
+                                        <div class="card border-secondary">
+                                            <div class="card-body py-2">
+                                                <div class="row">
+                                                    <div class="col-md-3 text-center px-0">
+                                                        <button class="btn btn-sm btn-secondary" onclick="blur_slider_correlation.val(0);blur_tooltip();">
+                                                            <i class="fas fa-eye-slash"></i>
+      										                <span class="label-icon">Hide</span>
+                                                        </button>
+                                                    </div>
+                                                    <div class="col-md-6 text-center pl-0 pt-1">
+                                                        <input type="range" min="0" max="15" step="0.1" value="10" id="blur-slider-correlation">
+                                                    </div>
+                                                    <div class="col-md-3 text-center">
+                                                        <button class="btn btn-sm btn-secondary" onclick="blur_slider_correlation.val(15);blur_tooltip();">
+                                                            <i class="fas fa-image"></i>
+                                                            <span class="label-icon">Full</span>
+                                                        </button>
+                                                    </div>
+                                                </div>
+                                            </div>
+                                        </div>
+                                    </span>
+{#                                    <span class="float-right mx-1">#}
+{#										{% if dict_object["object_type"] != "" %}#}
+{#											{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}#}
+{#                                                {% include 'import_export/block_add_user_object_to_export.html' %}#}
+{#                                            {% endwith %}#}
+{#										{% endif %}#}
+{#                                    </span>#}
+                                    <span class="float-right mt-2">
+										<button class="btn btn-primary py-1" onclick="resize_graph();">
+											<i class="fas fa-sync"></i>&nbsp;Resize Graph
+										</button>
+									</span>
+                                    <span class="float-right mt-2 mx-1">
+										<button class="btn btn-primary py-1" onclick="reset_graph();">
+											<i class="fas fa-undo"></i>&nbsp;Reset Graph
+										</button>
+									</span>
+                                    <div id="incomplete_graph" class="text-danger mt-3">
+                                        <i class="fas fa-exclamation-triangle"></i>&nbsp;Graph Incomplete, Max Nodes Reached.
+                                    </div>
+								</div>
+								<div class="card-body graph_panel">
+									<div id="graph_loading" class="ml-3 mt-3">
+										<div class="spinner-border text-secondary" role="status">
+											<span class="sr-only">Loading...</span>
+										</div>
+										<span>Loading...</span>
+									</div>
+									<div id="graph">
+									</div>
+								</div>
+							</div>
+
+{#                            <p>Press <b>H</b> on an object / node to hide it.</p>#}
+{#                            {% if dict_object["hidden"] %}#}
+{#                                <h5>Hidden objects:</h5>#}
+{#                                {% for obj_hidden in dict_object["hidden"] %}#}
+{#                                    {{ obj_hidden }} <br>#}
+{#                                {% endfor %}#}
+{#                            {% endif %}#}
+
+						</div>
+
+						<div class="col-xl-2">
+
+							<div class="card mb-3">
+								<div class="card-body text-center px-0 py-0">
+
+									<ul class="list-group">
+                                        <li class="list-group-item list-group-item-info">Relationship</li>
+                                        <form action="{{ url_for('correlation.show_relationship') }}" method="post">
+
+                                            <input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
+                                            <input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
+                                            <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
+
+{#                                        <li class="list-group-item text-left">#}
+{#                                                <div class="form-check">#}
+{#                                                    <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
+{#                                                    <label class="form-check-label" for="forwardCheck">Forwards</label>#}
+{#                                                </div>#}
+{#                                                <div class="form-check">#}
+{#                                                    <input class="form-check-input" type="checkbox" value="True" id="mentionCheck" name="mentionCheck" {%if "mention" in dict_object["filter"]%}checked{%endif%}>#}
+{#                                                    <label class="form-check-label" for="mentionCheck">Mentions</label>#}
+{#                                                </div>#}
+{##}
+{#                                        </li>#}
+                                        <li class="list-group-item text-left">
+
+                                            <div class="form-group">
+                                                <label for="max_nb_nodes_in">Correlation Depth:</label>
+                                                <input class="form-control" type="number" value="{{dict_object["level"]}}" min="0" id="level" name="level">
+                                            </div>
+
+
+                                        </li>
+                                        <li class="list-group-item text-left">
+
+                                            <div class="form-group">
+                                                <label for="max_nb_nodes_in">Max number of nodes:</label>
+                                                <input class="form-control" type="number" value="{{dict_object["max_nodes"]}}" min="0" id="max_nb_nodes_in" name="max_nb_nodes_in">
+                                            </div>
+
+                                            <div class="text-center">
+                                                <input class="btn btn-primary" type="submit" value="Redraw Graph">
+                                            </div>
+
+
+                                        </li>
+                                        </form>
+									</ul>
+
+									<ul class="list-group">
+                                        <li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
+                                        <li class="list-group-item text-left">
+                                            <p>Double click on a node to open this object<br><br>
+                                                <svg height="26" width="26">
+                                                    <g class="nodes">
+                                                        <circle cx="13" cy="13" r="13" fill="orange"></circle>
+													</g>
+												</svg>
+                                                Current Object<br>
+                                            </p>
+                                        </li>
+                                    </ul>
+                                    <ul class="list-group">
+                                        <li class="list-group-item list-group-item-secondary"><i class="fas fa-project-diagram"></i> Direct Relationships</li>
+                                        <li class="list-group-item text-left">
+                                            {% for relationship in  dict_object['nb_relation'] %}
+                                                <div class="row">
+                                                    <div class="col-8">
+                                                        {{ relationship }}
+                                                    </div>
+                                                    <div class="col-4">
+                                                        <span class="badge badge-primary">{{ dict_object['nb_relation'][relationship] }}</span>
+                                                    </div>
+                                                </div>
+                                            {% endfor %}
+                                        </li>
+                                    </ul>
+
+								</div>
+							</div>
+						</div>
+					</div>
+
+					<div class="row">
+						<div class="col-xl-10">
+
+							{% include 'correlation/legend_graph_correlation.html' %}
+
+						</div>
+					</div>
+
+
+
+                    <div class="card my-2">
+                        <div class="card-header bg-light">
+                            <h4><i class="fas fa-tags"></i> Tags All Objects</h4>
+                        </div>
+                        <div class="card-body">
+                            <form action="{{ url_for('correlation.correlation_tags_add') }}" method='post' onsubmit="SubmitAddTags();">
+                                <input type="hidden" id="tag_obj_type" name="tag_obj_type" value="{{ dict_object["object_type"] }}">
+                                <input type="hidden" id="tag_subtype" name="tag_subtype" value="{{ dict_object["metadata"]["type_id"] }}">
+                                <input type="hidden" id="tag_obj_id" name="tag_obj_id" value="{{ dict_object["correlation_id"] }}">
+                                <input type="hidden" id="tag_level" name="tag_level" value="{{dict_object["level"]}}">
+                                <input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
+                                <input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
+                                <input type="hidden" id="tag_hidden" name="tag_hidden" value="{{dict_object["hidden_str"]}}">
+                                {% include 'tags/block_tags_selector.html' %}
+                                <button class="btn btn-primary mt-2">
+									<i class="fas fa-tag"></i> Add Tags
+                                </button>
+                            </form>
+                        </div>
+                    </div>
+
+				</div>
+			</div>
+		</div>
+
+<script>
+
+var all_graph = {};
+$(document).ready(function(){
+        $("#incomplete_graph").hide();
+		$("#page-Decoded").addClass("active");
+
+		all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
+		all_graph.onResize();
+});
+
+const blur_slider_correlation = $('#blur-slider-correlation');
+function blur_tooltip(){
+    var image = $('#tooltip_screenshot_correlation')[0];
+    if (image) {
+        var blurValue = $('#blur-slider-correlation').val();
+        blurValue = 15 - blurValue;
+        image.style.filter = "blur(" + blurValue + "px)";
+    }
+}
+
+$(window).on("resize", function() {
+		all_graph.onResize();
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+function SubmitAddTags() {
+	var tags = ltags.getValue();
+	var tagsgalaxy = ltagsgalaxies.getValue();
+	$('#ltags').val(tags);
+	$('#ltagsgalaxies').val(tagsgalaxy);
+	return true;
+}
+</script>
+
+<script>
+	function resize_graph() {
+		zoom.translateTo(svg_node, 200, 200);
+		zoom.scaleTo(svg_node, 2);
+	}
+
+    function reset_graph() {
+        window.location.href = "{{ url_for('correlation.show_relationship') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}"
+    }
+
+</script>
+
+<script>
+var correl_link = "{{ url_for('correlation.show_relationship') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}&max_nodes={{ dict_object["max_nodes"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}"
+{% if 'hidden_str' in dict_object %}
+    correl_link = correl_link + "&hidden={{ dict_object["hidden_str"] }}"
+{% endif %}
+
+var width = 400,
+		height = 400;
+
+var link;
+
+var zoom = d3.zoom()
+	.scaleExtent([.2, 10])
+	.on("zoom", zoomed);
+
+// Loading:
+//$("#graph_loading").show()
+
+//var transform = d3.zoomIdentity;
+
+var color = d3.scaleOrdinal(d3.schemeCategory10);
+
+var div = d3.select("body").append("div")
+		.attr("class", "tooltip_graph")
+		.style("opacity", 0);
+
+var simulation = d3.forceSimulation()
+		.force("link", d3.forceLink().id(function(d) { return d.id; }))
+		.force("charge", d3.forceManyBody())
+		.force("center", d3.forceCenter(width / 2, height / 2));
+		//.on("tick", ticked);
+
+var currentObject = null;
+
+var svg_node = d3.select("#graph").append("svg")
+		.attr("id", "graph_div")
+		.attr("width", width)
+		.attr("height", height)
+		.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
+		.on("dblclick.zoom", null)
+
+var container_graph = svg_node.append("g");
+		//.attr("transform", "translate(40,0)")
+		//.attr("transform", "scale(2)");
+
+
+function create_graph(url){
+
+d3.json(url)
+.then(function(data){
+
+    // ----------------------------------------------------------------------------------
+    svg_node.append("defs").selectAll("marker")
+        .data(["end"])      // Different link/path types can be defined here
+    .enter().append("marker")
+        .attr("id", String)
+        .attr("viewBox", "0 -5 10 10")
+        .attr("refX", 25)   // Controls the distance of the marker from the node
+        .attr("refY", 0)
+        .attr("markerWidth", 6)
+        .attr("markerHeight", 6)
+        .attr("orient", "auto")
+    .append("path")
+        .attr("d", "M0,-5L10,0L0,5");
+    // ----------------------------------------------------------------------------------
+
+	var link = container_graph.append("g")
+        /*.selectAll("line")
+        .data(data.links)
+        .enter().append("line")
+            .attr("class", "link")
+            .attr("marker-end", "url(#end)");*/
+        .selectAll("g")
+        .data(data.links)
+        .enter().append("g");
+    link.append("line")
+        .attr("class", "link")
+        .attr("marker-end", "url(#end)")
+    link.append("text")
+        .attr("class", "link-label")
+        .style("font-size", "6px")
+        .text(function(d) { return d.relationship; })
+
+
+		//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
+
+	var node = container_graph.selectAll(".node")
+				.data(data.nodes)
+			.enter().append("g")
+			.attr("class", "nodes")
+			.on("dblclick", doubleclick)
+			.on("click", click)
+			.on("mouseover", mouseovered)
+			.on("mouseout", mouseouted)
+			.call(d3.drag()
+					.on("start", drag_start)
+					.on("drag", dragged)
+					.on("end", drag_end));
+
+    /*
+    node.append("image")
+    .attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png")
+    .attr("height", 20)
+    .attr("width", 20)
+    .attr("x", -10)
+    .attr("y", -10);
+
+     */
+
+	node.append("circle")
+			.attr("r", function(d) {
+				return (d.style.node_radius);})
+			.attr("fill", function(d) {
+					return d.style.node_color;});
+
+	node.append('text')
+		  .attr('text-anchor', 'middle')
+		  .attr('dominant-baseline', 'central')
+		  .attr("class", function(d) {
+				return "graph_node_icon " + d.style.icon_class
+			})
+    	.attr('font-size', '8px' )
+			.attr('pointer-events', 'none')
+    	.text(function(d) {
+				//if(d.hash){
+					return d.style.icon_text
+				//}
+			});
+
+	zoom.translateTo(svg_node, 200, 200);
+	zoom.scaleTo(svg_node, 2);
+
+/*	node.append("text")
+		      .attr("dy", 3)
+					.attr("dx", 7)
+					.attr("class", "graph_text_node")
+		      //.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
+		      .text(function(d) { return d.id; });*/
+
+	simulation
+			.nodes(data.nodes)
+			.on("tick", ticked);
+
+	simulation.force("link")
+			.links(data.links);
+
+	function ticked() {
+		link.select("line")
+				.attr("x1", function(d) { return d.source.x; })
+				.attr("y1", function(d) { return d.source.y; })
+				.attr("x2", function(d) { return d.target.x; })
+				.attr("y2", function(d) { return d.target.y; });
+
+        link.select("text")
+            .attr("x", function(d) {
+                return (d.source.x + d.target.x) / 2;
+            })
+            .attr("y", function(d) {
+                return (d.source.y + d.target.y) / 2;
+            });
+
+		/*node
+				.attr("cx", function(d) { return d.x; })
+				.attr("cy", function(d) { return d.y; });*/
+		node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
+	}
+
+	// Loading ...
+	$("#graph_loading").remove();
+
+    if (!data.meta.complete){
+        $("#incomplete_graph").show();
+    }
+
+    d3.select("body").on("keypress", keypressed)
+
+
+
+})
+.catch(function(error) {
+				$("#graph_loading").remove()
+				svg_node.remove();
+				d3.select("#graph").append("div")
+													 .text(error);
+		});
+}
+
+function zoomed() {
+	container_graph.attr("transform", d3.event.transform);
+}
+
+function doubleclick (d) {
+	window.open(d.url, '_blank');
+}
+
+function keypressed () {
+    //console.log(d3.event.keyCode)
+    //console.log(currentObject.id)
+    // hide node, H or h key
+    if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
+        window.location.href = correl_link + "&hide=" + currentObject.id
+    }
+
+}
+
+function click (d) {
+console.log('clicked')
+}
+
+function drag_start(d) {
+if (!d3.event.active) simulation.alphaTarget(0.3).restart();
+d.fx = d.x;
+d.fy = d.y;
+}
+
+function dragged(d) {
+d.fx = d3.event.x;
+d.fy = d3.event.y;
+}
+
+function drag_end(d) {
+if (!d3.event.active) simulation.alphaTarget(0);
+d.fx = d.x;
+d.fy = d.y;
+}
+
+function mouseovered(d) {
+
+currentObject = d;
+
+var d3_pageX = d3.event.pageX;
+var d3_pageY = d3.event.pageY;
+
+if (d.popover) {
+	div.html(d.popover)
+		.style("left", (d3_pageX) + "px")
+		.style("top", (d3_pageY - 28) + "px");
+
+	div.transition()
+		.duration(200)
+		.style("opacity", 1);
+    blur_tooltip();
+} else {
+
+		var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
+		var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
+
+		div.html(pop_header + spinner)
+			.style("left", (d3_pageX) + "px")
+			.style("top", (d3_pageY - 28) + "px");
+
+		div.transition()
+			.duration(200)
+			.style("opacity", 1);
+
+		$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
+				function(data){
+					var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
+					Object.keys(data).forEach(function(key) {
+						if (key=="status") {
+							desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
+							if (data["status"]) {
+								desc = desc + "Green"
+							} else {
+								desc = desc + "Red"
+							}
+							desc = desc + ";\"><i class=\"fas "
+							if (data["status"]) {
+								desc = desc + "fa-check-circle\"></i>UP"
+							} else {
+								desc = desc + "fa-times-circle\"></i>DOWN"
+							}
+							desc = desc + "</div></dd>"
+						} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
+							desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+						}
+					});
+					desc = desc + "</dl>"
+
+                    if (data["tags"]) {
+						data["tags"].forEach(function(tag) {
+							desc = desc + "<span class=\"badge badge-warning\">"+ sanitize_text(tag) +"</span>";
+						});
+					}
+
+					if (data["img"]) {
+                        if (data["tags_safe"]) {
+                            if (data["type"] === "screenshot") {
+						        desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
+                            } else {
+                                desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
+                            }
+                            desc = desc + data["img"] +" class=\"img-thumbnail blured\" id=\"tooltip_screenshot_correlation\" style=\"\"/>";
+                        } else {
+                            desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
+                        }
+					}
+
+					desc = desc + "</div></div>"
+					div.html(desc)
+					 	.style("left", (d3_pageX) + "px")
+					 	.style("top", (d3_pageY - 28) + "px");
+					d.popover = desc
+
+                    if (data["img"]) {
+                        blur_tooltip();
+                    }
+
+				})
+				.fail(function(error) {
+					desc = pop_header + "<div class=\"card-body bg-dark pt-0\"><i class=\"fas fa-3x fa-times text-danger\"></i>"+ error.statusText +"</div>"
+					div.html(desc)
+					 	.style("left", (d3_pageX) + "px")
+					 	.style("top", (d3_pageY - 28) + "px");
+				});
+		}
+}
+
+function mouseouted() {
+    currentObject = null;
+
+	div.transition()
+		.duration(500)
+		.style("opacity", 0);
+        //link.select("line")
+}
+
+all_graph.onResize = function () {
+	var aspect = 1000 / 500, all_graph = $("#graph_div");
+	var targetWidth = all_graph.parent().width();
+	all_graph.attr("width", targetWidth);
+	all_graph.attr("height", targetWidth / aspect);
+}
+
+window.all_graph = all_graph;
+</script>
+
+        </body>
+
+</html>