chg: [relationship] get object relationhips by relationship name + object types

bin/importer/feeders/abstract_chats_feeder.py

@@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
 import datetime
 import os
 import sys
+import time
 
 from abc import ABC
 
@@ -164,6 +165,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             self.obj = Messages.Message(obj_id)
         return self.obj
 
+    # TODO handle subchannel
+    def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
+        chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
+
+        # Obj Daterange
+        chat.add(date)
+
+        if meta_chat.get('name'):
+            chat.set_name(meta_chat['name'])
+
+        if meta_chat.get('info'):
+            chat.set_info(meta_chat['info'])
+
+        if meta_chat.get('date'): # TODO check if already exists
+            chat.set_created_at(int(meta_chat['date']['timestamp']))
+
+        if meta_chat.get('icon'):
+            img = Images.create(meta_chat['icon'], b64=True)
+            img.add(date, chat)
+            chat.set_icon(img.get_global_id())
+            if new_objs:
+                new_objs.add(img)
+
+        if meta_chat.get('username'):
+            username = Username(meta_chat['username'], self.get_chat_protocol())
+            chat.update_username_timeline(username.get_global_id(), int(time.time()))
+            username.add(date, obj=chat)  # TODO TODAY DATE
+
+        return chat
+
+    ##############################################################################################################################
+
+
     def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
         meta = self.json_data['meta']['chat'] # todo replace me by function
         chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
@@ -365,73 +399,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             # CHAT
             chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
-            # # TODO HANDLE OTHERS OBJECT TYPE
-            # # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-            # # Message forward + Discussion
-            # if self.get_json_meta().get('forward'):
-            #     discussion_id = self.get_json_meta().get('discussion')
-            #     forward_from = self.get_message_forward()
-            #
-            #     if discussion_id:       # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
-            #         chat_forward_id = forward_from['from']['id']
-            #         message_forward_id = forward_from['from']['channel_post']
-            #
-            #         # if chat_forward_id == discussion_id:
-            #         #     linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
-            #         #     if linked_chat.exists():
-            #         #         # create thread
-            #         #         # add message replies for each childrens
-            #
-            # # TODO HANDLE THREAD
-            # # TODO Change FORWARD META FIELDS
-            # # meta['forward'] = {}
-            # #       # CHAT ID
-            # #       # SUBCHANNEL ID     -> can be None
-            # #       # Message ID
-            #
-            # # meta['forward']['origin']
-            # #       # same as 'forward'
-            #
-            # if self.get_json_meta().get('forward'):
-            #     forward = self.get_message_forward()
-            #     f_chat = forward['chat']
-            #     f_subchannel = forward.get('subchannel')
-            #     f_id = forward.get('id')
-            #     if not f_subchannel:
-            #         chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
-            #         if chat_forward.exists():
-            #             for chat_obj in chat_objs:
-            #                 if chat_obj.type == 'chat':
-            #                     chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
-            #             # TODO LIST FORWARDED MESSAGES
-            #
-            #
-            # # Discord -> serverID + subchannel ID + message ID
-            # # Telegram -> chat ID + Message ID
-            # #                 + ORIGIN IDs
-            #
-            #
-            #
-            # # TODO create relationships graph
-            #
-            #
-            # # TODO REMOVE ME
-            # # Message forward  # TODO handle subchannel + message ID
-            # if self.get_json_meta().get('forward'):
-            #     forward_from = self.get_message_forward()
-            #     print('-----------------------------------------------------------')
-            #     print(forward_from)
-            #     if forward_from:
-            #         forward_from_type = forward_from['from']['type']
-            #         if forward_from_type == 'channel' or forward_from_type == 'chat':
-            #             chat_forward_id = forward_from['from']['id']
-            #             chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
-            #             if chat_forward.exists():
-            #                 for chat_obj in chat_objs:
-            #                     if chat_obj.type == 'chat':
-            #                         chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
-            #                         # chat_forward.add_relationship(obj.get_global_id(), 'forward')
-
             # SENDER # TODO HANDLE NULL SENDER
             user_account = self.process_sender(new_objs, obj, date, timestamp)
 
@@ -449,4 +416,24 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
                 #       -> subchannel ?
                 #       -> thread id ?
 
+
+        #######################################################################
+
+        ## FORWARD ##
+        # chat_fwd = None
+        # if self.get_json_meta().get('forward'):
+        #     meta_fwd = self.get_message_forward()
+        #     if meta_fwd['chat']:
+        #         chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
+        #         for chat_obj in chat_objs:
+        #             if chat_obj.type == 'chat':
+        #                 chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward')
+        #
+        # # TODO chat_fwd -> message
+        # if chat_fwd:
+        #     for obj in objs:
+        #         if obj.type == 'message':
+        #             chat_fwd.add_relationship(obj.get_global_id(), 'forward')
+        # -FORWARD- #
+
         return new_objs | objs

bin/lib/objects/ail_objects.py

@@ -562,15 +562,22 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
 
 # --- CORRELATION --- #
 
+#### RELATIONSHIPS ####
+
+def get_relationships():
+    return relationships_engine.get_relationships()
+
+def sanitize_relationships(relationships):
+    return relationships_engine.sanitize_relationships(relationships)
 def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.get_nb_relationships(filter=filter_types)
 
-def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
                                  objs_hidden=set(),
                                  flask_context=False):
     obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
-    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
+    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
                                                                      filter_types=filter_types,
                                                                      max_nodes=max_nodes, level=level,
                                                                      objs_hidden=objs_hidden)
@@ -581,6 +588,9 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max
             "meta": meta}
 
 
+# --- RELATIONSHIPS --- #
+
+
 # if __name__ == '__main__':
     # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
     # r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),

bin/lib/relationships_engine.py

@@ -16,23 +16,72 @@ config_loader = None
 
 
 RELATIONSHIPS = {
-    "forward",
+    "forward",  # forwarded_to
     "mention"
 }
+
+RELATIONSHIPS_OBJS = {
+    "forward": {
+        'chat': {'chat', 'message'},
+        'message': {'chat'}
+    },
+    "mention": {}
+}
+
 def get_relationships():
     return RELATIONSHIPS
 
+def sanitize_relationships(relationships):
+    rels = get_relationships()
+    if relationships:
+        relationships = set(relationships).intersection(rels)
+    if not relationships:
+        relationships = rels
+        if not relationships:
+            return []
+    return relationships
 
-def get_obj_relationships_by_type(obj_global_id, relationship):
+def get_relationship_obj_types(relationship):
-    return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
+    return RELATIONSHIPS_OBJS.get(relationship, {})
 
-def get_obj_nb_relationships_by_type(obj_global_id, relationship):
+def get_relationship_objs(relationship, obj_type):
-    return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
+    return get_relationship_obj_types(relationship).get(obj_type, set())
 
-def get_obj_relationships(obj_global_id):
+def sanityze_obj_types(relationship, obj_type, filter_types):
-    relationships = []
+    objs_types = get_relationship_objs(relationship, obj_type)
-    for relationship in get_relationships():
+    if filter_types:
-        for rel in get_obj_relationships_by_type(obj_global_id, relationship):
+        filter_types = objs_types.intersection(filter_types)
+    if not filter_types:
+        filter_types = objs_types
+        if not filter_types:
+            return []
+    return filter_types
+
+# TODO check obj_type
+# TODO sanitize relationships
+
+def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+    obj_type = obj_global_id.split(':', 1)[0]
+    relationships = {}
+    filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+    for o_type in filter_types:
+        relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
+    return relationships
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+    obj_type = obj_global_id.split(':', 1)[0]
+    relationships = {}
+    filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+    for o_type in filter_types:
+        relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
+    return relationships
+
+def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
+    all_relationships = []
+    for relationship in relationships:
+        obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
+        for obj_type in obj_relationships:
+            for rel in obj_relationships[obj_type]:
                 meta = {'relationship': relationship}
                 direction, obj_id = rel.split(':', 1)
                 if direction == 'i':
@@ -42,15 +91,12 @@ def get_obj_relationships(obj_global_id):
                     meta['target'] = obj_id
                     meta['source'] = obj_global_id
 
-            if not obj_id.startswith('chat'):
-                continue
-
                 meta['id'] = obj_id
                 # meta['direction'] = direction
-            relationships.append(meta)
+                all_relationships.append(meta)
-    return relationships
+    return all_relationships
 
-def get_obj_nb_relationships(obj_global_id):
+def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
     nb = {}
     for relationship in get_relationships():
         nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
@@ -59,27 +105,27 @@ def get_obj_nb_relationships(obj_global_id):
 
 # TODO Filter by obj type ???
 def add_obj_relationship(source, target, relationship):
-    r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
+    source_type = source.split(':', 1)[0]
-    r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
+    target_type = target.split(':', 1)[0]
-    # r_rel.sadd(f'rels:{source}', relationship)
+    r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
-    # r_rel.sadd(f'rels:{target}', relationship)
+    r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
 
 
-def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
     links = []
     nodes = set()
     meta = {'complete': True, 'objs': set()}
     done = set()
     done_link = set()
 
-    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
     return nodes, links, meta
 
-def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
     meta['objs'].add(obj_global_id)
     nodes.add(obj_global_id)
 
-    for rel in get_obj_relationships(obj_global_id):
+    for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
         meta['objs'].add(rel['id'])
 
         if rel['id'] in done:
@@ -99,13 +145,7 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
         if level > 0:
             next_level = level - 1
 
-            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
 
     # done.add(rel['id'])
 
-
-if __name__ == '__main__':
-    source = ''
-    target = ''
-    add_obj_relationship(source, target, 'forward')
-    # print(get_obj_relationships(source))

var/www/blueprints/correlation.py

@@ -263,7 +263,10 @@ def relationships_graph_node_json():
     max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
     level = sanitise_level(request.args.get('level'))
 
-    json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
+    filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
+    relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
+
+    json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
     return jsonify(json_graph)
 
 
@@ -278,8 +281,28 @@ def show_relationship():
         max_nodes = request.form.get('max_nb_nodes_in')
         level = sanitise_level(request.form.get('level'))
 
+        ## get all selected relationships
+        relationships = []
+        for relationship in ail_objects.get_relationships():
+            rel_option = request.form.get(f'relationship_{relationship}_Check')
+            if rel_option:
+                relationships.append(relationship)
+
+        relationships = ",".join(relationships)
+
+        ## get all selected objects types
+        filter_types = []
+        for ob_type in ail_objects.get_all_objects():
+            correl_option = request.form.get(f'{ob_type}_Check')
+            if correl_option:
+                filter_types.append(ob_type)
+
+        # list as params
+        filter_types = ",".join(filter_types)
+
         # redirect to keep history and bookmark
         return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
+                                filter=filter_types, relationships=relationships,
                                 max_nodes=max_nodes, level=level))
 
     # request.method == 'GET'
@@ -290,6 +313,9 @@ def show_relationship():
         max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
         level = sanitise_level(request.args.get('level'))
 
+        filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
+        relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
+
         # check if obj_id exist
         if not ail_objects.exists_obj(obj_type, subtype, obj_id):
             return abort(404)
@@ -300,6 +326,8 @@ def show_relationship():
                            "object_type": obj_type,
                            "max_nodes": max_nodes, "level": level,
                            "correlation_id": obj_id,
+                           "relationships": relationships, "relationships_str": ",".join(relationships),
+                           "filter": filter_types, "filter_str": ",".join(filter_types),
                            "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
                            "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
                            }

var/www/templates/correlation/show_relationship.html

@@ -205,13 +205,37 @@
 								<div class="card-body text-center px-0 py-0">
 
 									<ul class="list-group">
-                                        <li class="list-group-item list-group-item-info">Relationship</li>
+                                        <li class="list-group-item list-group-item-info">Relationships</li>
                                         <form action="{{ url_for('correlation.show_relationship') }}" method="post">
 
+                                            <li class="list-group-item text-left">
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_forward_Check" name="relationship_forward_Check" {%if "forward" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_forward_Check">Forward</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_mention_Check">Mention</label>
+                                                </div>
+                                            </li>
+
                                             <input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
                                             <input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
                                             <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
 
+                                            <li class="list-group-item list-group-item-info">Objects Types</li>
+
+                                            <li class="list-group-item text-left">
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="chat_Check">Chat</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="message_Check">Message</label>
+                                                </div>
+                                            </li>
+
 {#                                        <li class="list-group-item text-left">#}
 {#                                                <div class="form-check">#}
 {#                                                    <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
@@ -324,7 +348,7 @@ $(document).ready(function(){
         $("#incomplete_graph").hide();
 		$("#page-Decoded").addClass("active");
 
-		all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
+		all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
 		all_graph.onResize();
 });