diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 881cb8a9..34ccca8c 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -100,14 +100,15 @@ class FeederImporter(AbstractImporter): else: objs = set() - objs.add(data_obj) + if data_obj: + objs.add(data_obj) for obj in objs: if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() return obj, f'{feeder_name} {gzip64_content}' else: # Messages save on DB - if obj.exists(): + if obj.exists() and obj.type != 'chat': return obj, f'{feeder_name}' @@ -136,4 +137,5 @@ class FeederModuleImporter(AbstractModule): # Launch Importer if __name__ == '__main__': module = FeederModuleImporter() + # module.debug = True module.run() diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 1dadb970..6b8f1041 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -93,7 +93,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta'].get('reactions', []) def get_message_timestamp(self): - return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + if not self.json_data['meta'].get('date'): + return None + else: + return self.json_data['meta']['date']['timestamp'] # if self.json_data['meta'].get('date'): # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) # date = date.strftime('%Y/%m/%d') @@ -115,17 +118,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC): def get_message_reply_id(self): return self.json_data['meta'].get('reply_to', {}).get('message_id') + def get_message_forward(self): + return self.json_data['meta'].get('forward') + def get_message_content(self): decoded = base64.standard_b64decode(self.json_data['data']) return _gunzip_bytes_obj(decoded) - def get_obj(self): # TODO handle others objects -> images, pdf, ... + def get_obj(self): #### TIMESTAMP #### timestamp = self.get_message_timestamp() #### Create Object ID #### chat_id = self.get_chat_id() - message_id = self.get_message_id() + try: + message_id = self.get_message_id() + except KeyError: + if chat_id: + self.obj = Chat(chat_id, self.get_chat_instance_uuid()) + return self.obj + else: + self.obj = None + return None + thread_id = self.get_thread_id() # channel id # thread id @@ -236,7 +251,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # # ADD NEW MESSAGE REF (used by discord) def process_sender(self, new_objs, obj, date, timestamp): - meta = self.json_data['meta']['sender'] + meta = self.json_data['meta'].get('sender') + if not meta: + return None + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) # date stat + correlation @@ -286,8 +304,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # REPLY reply_id = self.get_message_reply_id() - # TODO Translation - print(self.obj.type) # TODO FILES + FILES REF @@ -295,7 +311,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # get object by meta object type if self.obj.type == 'message': # Content - obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation + obj = Messages.create(self.obj.id, self.get_message_content()) # FILENAME media_name = self.get_media_name() @@ -305,7 +321,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC): for reaction in self.get_reactions(): obj.add_reaction(reaction['reaction'], int(reaction['count'])) - + elif self.obj.type == 'chat': + pass else: chat_id = self.get_chat_id() thread_id = self.get_thread_id() @@ -341,12 +358,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # CHAT chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) + # Message forward + # if self.get_json_meta().get('forward'): + # forward_from = self.get_message_forward() + # print('-----------------------------------------------------------') + # print(forward_from) + # if forward_from: + # forward_from_type = forward_from['from']['type'] + # if forward_from_type == 'channel' or forward_from_type == 'chat': + # chat_forward_id = forward_from['from']['id'] + # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid()) + # if chat_forward.exists(): + # for chat_obj in chat_objs: + # if chat_obj.type == 'chat': + # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') + # # chat_forward.add_relationship(obj.get_global_id(), 'forward') + # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) + if user_account: # UserAccount---ChatObjects - for obj_chat in chat_objs: - user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) + for obj_chat in chat_objs: + user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) # if chat: # TODO Chat---Username correlation ??? # # Chat---Username => need to handle members and participants diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 040c3ea5..e0dffd9d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -76,6 +76,7 @@ class Chat(AbstractChatObject): meta['tags'] = self.get_tags(r_list=True) if 'icon' in options: meta['icon'] = self.get_icon() + meta['img'] = meta['icon'] if 'info' in options: meta['info'] = self.get_info() if 'participants' in options: @@ -93,6 +94,8 @@ class Chat(AbstractChatObject): if 'threads' in options: meta['threads'] = self.get_threads() print(meta['threads']) + if 'tags_safe' in options: + meta['tags_safe'] = self.is_tags_safe(meta['tags']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 2655c2ee..659047be 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -131,7 +131,7 @@ class Message(AbstractObject): if meta: _, user_account_subtype, user_account_id = user_account.split(':', 3) user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'}) - return user_account + return user_account def get_files_names(self): names = [] @@ -148,15 +148,32 @@ class Message(AbstractObject): def add_reaction(self, reactions, nb_reaction): r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction) - # Update value on import - # reply to -> parent ? - # reply/comment - > children ? + # Interactions between users -> use replies # nb views - # reactions - # nb fowards - # room ??? - # message from channel ??? + # MENTIONS -> Messages + Chats + # # relationship -> mention - Chat -> Chat + # - Message -> Chat + # - Message -> Message ??? fetch mentioned messages + # FORWARDS + # TODO Create forward CHAT -> message + # message (is forwarded) -> message (is forwarded from) ??? + # # TODO get source message timestamp + # + # # is forwarded + # # forwarded from -> check if relationship + # # nb forwarded -> scard relationship + # + # Messages -> CHATS -> NB forwarded + # CHAT -> NB forwarded by chats -> NB messages -> parse full set ???? + # + # + # + # + # + # + # show users chats # message media + # flag is deleted -> event or missing from feeder pass ??? def get_translation(self, content=None, source=None, target='fr'): """ diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index d651761f..86eacc44 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -24,6 +24,7 @@ from lib.ConfigLoader import ConfigLoader from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations +from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -284,6 +285,22 @@ class AbstractObject(ABC): ## -Correlation- ## + ## Relationship ## + + def get_nb_relationships(self, filter=[]): + return get_obj_nb_relationships(self.get_global_id()) + + def add_relationship(self, obj2_global_id, relationship, source=True): + # is source + if source: + print(self.get_global_id(), obj2_global_id, relationship) + add_obj_relationship(self.get_global_id(), obj2_global_id, relationship) + # is target + else: + add_obj_relationship(obj2_global_id, self.get_global_id(), relationship) + + ## -Relationship- ## + ## Parent ## def is_parent(self): diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 1b01c2c1..15717c78 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -10,6 +10,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ConfigLoader import ConfigLoader from lib.ail_core import get_all_objects, get_object_all_subtypes from lib import correlations_engine +from lib import relationships_engine from lib import btc_ail from lib import Tag @@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_ # --- CORRELATION --- # +def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]): + obj = get_object(obj_type, subtype, obj_id) + return obj.get_nb_relationships(filter=filter_types) + +def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, + objs_hidden=set(), + flask_context=False): + obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) + nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, + filter_types=filter_types, + max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden) + # print(meta) + meta['objs'] = list(meta['objs']) + return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context), + "links": links, + "meta": meta} + # if __name__ == '__main__': # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}]) diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py new file mode 100755 index 00000000..6791214a --- /dev/null +++ b/bin/lib/relationships_engine.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +r_rel = config_loader.get_db_conn("Kvrocks_Relationships") +config_loader = None + + +RELATIONSHIPS = { + "forward", + "mention" +} +def get_relationships(): + return RELATIONSHIPS + + +def get_obj_relationships_by_type(obj_global_id, relationship): + return r_rel.smembers(f'rel:{relationship}:{obj_global_id}') + +def get_obj_nb_relationships_by_type(obj_global_id, relationship): + return r_rel.scard(f'rel:{relationship}:{obj_global_id}') + +def get_obj_relationships(obj_global_id): + relationships = [] + for relationship in get_relationships(): + for rel in get_obj_relationships_by_type(obj_global_id, relationship): + meta = {'relationship': relationship} + direction, obj_id = rel.split(':', 1) + if direction == 'i': + meta['source'] = obj_id + meta['target'] = obj_global_id + else: + meta['target'] = obj_id + meta['source'] = obj_global_id + + if not obj_id.startswith('chat'): + continue + + meta['id'] = obj_id + # meta['direction'] = direction + relationships.append(meta) + return relationships + +def get_obj_nb_relationships(obj_global_id): + nb = {} + for relationship in get_relationships(): + nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship) + return nb + + +# TODO Filter by obj type ??? +def add_obj_relationship(source, target, relationship): + r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}') + r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}') + # r_rel.sadd(f'rels:{source}', relationship) + # r_rel.sadd(f'rels:{target}', relationship) + + +def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()): + links = [] + nodes = set() + meta = {'complete': True, 'objs': set()} + done = set() + done_link = set() + + _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + return nodes, links, meta + +def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()): + meta['objs'].add(obj_global_id) + nodes.add(obj_global_id) + + for rel in get_obj_relationships(obj_global_id): + meta['objs'].add(rel['id']) + + if rel['id'] in done: + continue + + if len(nodes) > max_nodes != 0: + meta['complete'] = False + break + + nodes.add(rel['id']) + + str_link = f"{rel['source']}{rel['target']}{rel['relationship']}" + if str_link not in done_link: + links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']}) + done_link.add(str_link) + + if level > 0: + next_level = level - 1 + + _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + + # done.add(rel['id']) + + +if __name__ == '__main__': + source = '' + target = '' + add_obj_relationship(source, target, 'forward') + # print(get_obj_relationships(source)) diff --git a/configs/6383.conf b/configs/6383.conf index a06d4e69..dfc1f205 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -664,6 +664,7 @@ namespace.db ail_datas namespace.dup ail_dups namespace.obj ail_objs namespace.tl ail_tls +namespace.rel ail_rels namespace.stat ail_stats namespace.tag ail_tags namespace.track ail_trackers diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 0e0b900f..f93ec08a 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -196,6 +196,11 @@ host = localhost port = 6383 password = ail_objs +[Kvrocks_Relationships] +host = localhost +port = 6383 +password = ail_rels + [Kvrocks_Timeline] host = localhost port = 6383 diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 4cf9ce53..2b81d96f 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -203,7 +203,7 @@ def get_description(): return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 # object exist else: - res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'tags_safe'}, + res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'}, flask_context=True) if 'tags' in res: res['tags'] = list(res['tags']) @@ -292,3 +292,64 @@ def correlation_tags_add(): max_nodes=nb_max, hidden=hidden, hidden_str=",".join(hidden), filter=",".join(filter_types))) + +##################################################################################### + +@correlation.route('/relationships/graph_node_json') +@login_required +@login_read_only +def relationships_graph_node_json(): + obj_id = request.args.get('id') + subtype = request.args.get('subtype') + obj_type = request.args.get('type') + max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) + level = sanitise_level(request.args.get('level')) + + json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True) + return jsonify(json_graph) + + +@correlation.route('/relationship/show', methods=['GET', 'POST']) +@login_required +@login_read_only +def show_relationship(): + if request.method == 'POST': + object_type = request.form.get('obj_type') + subtype = request.form.get('subtype') + obj_id = request.form.get('obj_id') + max_nodes = request.form.get('max_nb_nodes_in') + level = sanitise_level(request.form.get('level')) + + # redirect to keep history and bookmark + return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id, + max_nodes=max_nodes, level=level)) + + # request.method == 'GET' + else: + obj_type = request.args.get('type') + subtype = request.args.get('subtype', '') + obj_id = request.args.get('id') + max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) + level = sanitise_level(request.args.get('level')) + + # check if obj_id exist + if not ail_objects.exists_obj(obj_type, subtype, obj_id): + return abort(404) + # object exist + else: # TODO remove old dict key + dict_object = {"type": obj_type, + "id": obj_id, + "object_type": obj_type, + "max_nodes": max_nodes, "level": level, + "correlation_id": obj_id, + "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True), + "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id) + } + if subtype: + dict_object["subtype"] = subtype + dict_object["metadata"]['type_id'] = subtype + else: + dict_object["subtype"] = '' + dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id) + return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label, + tags_selector_data=Tag.get_tags_selector_data()) diff --git a/var/www/templates/correlation/metadata_card_chat.html b/var/www/templates/correlation/metadata_card_chat.html new file mode 100644 index 00000000..4e672e1a --- /dev/null +++ b/var/www/templates/correlation/metadata_card_chat.html @@ -0,0 +1,78 @@ + + + +{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#} +{# {% include 'modals/add_tags.html' %}#} +{#{% endwith %}#} + +{% include 'modals/edit_tag.html' %} + +
+
+

{{ dict_object["correlation_id"] }}

+ {{ dict_object }} +
{{ dict_object["correlation_id"] }}
+ + + {% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index cda58f1c..4005e13c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -541,6 +541,15 @@ d3.json(url) .on("drag", dragged) .on("end", drag_end)); + /* + node.append("image") + .attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png") + .attr("height", 20) + .attr("width", 20) + .attr("x", -10) + .attr("y", -10); + + */ node.append("circle") .attr("r", function(d) { diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html new file mode 100644 index 00000000..bff41724 --- /dev/null +++ b/var/www/templates/correlation/show_relationship.html @@ -0,0 +1,719 @@ + + + + + + + AIL - framework + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ + {% if dict_object["object_type"] == "pgp" %} + {% include 'correlation/metadata_card_pgp.html' %} + {% elif dict_object["object_type"] == "cryptocurrency" %} + {% include 'correlation/metadata_card_cryptocurrency.html' %} + {% elif dict_object["object_type"] == "username" %} + {% include 'correlation/metadata_card_username.html' %} + {% elif dict_object["object_type"] == "decoded" %} + {% include 'correlation/metadata_card_decoded.html' %} + {% elif dict_object["object_type"] == "chat" %} + {% include 'correlation/metadata_card_chat.html' %} + {% elif dict_object["object_type"] == "cve" %} + {% include 'correlation/metadata_card_cve.html' %} + {% elif dict_object["object_type"] == "domain" %} + {% include 'correlation/metadata_card_domain.html' %} + {% elif dict_object["object_type"] == "screenshot" %} + {% include 'correlation/metadata_card_screenshot.html' %} + {% elif dict_object["object_type"] == "title" %} + {% include 'correlation/metadata_card_title.html' %} + {% elif dict_object["object_type"] == "cookie-name" %} + {% include 'correlation/metadata_card_cookie_name.html' %} + {% elif dict_object["object_type"] == "etag" %} + {% include 'correlation/metadata_card_etag.html' %} + {% elif dict_object["object_type"] == "hhhash" %} + {% include 'correlation/metadata_card_hhhash.html' %} + {% elif dict_object["object_type"] == "item" %} + {% include 'correlation/metadata_card_item.html' %} + {% endif %} + +
+
+ +
+
+ Graph + +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ +{# #} +{# {% if dict_object["object_type"] != "" %}#} +{# {% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}#} +{# {% include 'import_export/block_add_user_object_to_export.html' %}#} +{# {% endwith %}#} +{# {% endif %}#} +{# #} + + + + + + +
+  Graph Incomplete, Max Nodes Reached. +
+
+
+
+ + Loading... +
+
+
+
+
+ +{#

Press H on an object / node to hide it.

#} +{# {% if dict_object["hidden"] %}#} +{#
Hidden objects:
#} +{# {% for obj_hidden in dict_object["hidden"] %}#} +{# {{ obj_hidden }}
#} +{# {% endfor %}#} +{# {% endif %}#} + +
+ +
+ +
+
+ +
    +
  • Relationship
    • +
    + + + + + +{#
  • #} +{#
    #} +{# #} +{# #} +{#
    #} +{#
    #} +{# #} +{# #} +{#
    #} +{##} +{#
    • #} +
  • + +
    + + +
    + + +
    • +
  • + +
    + + +
    + +
    + +
    + + +
    • +
    +
+ +
    +
    • +
  • +

    Double click on a node to open this object

    + + + + + + Current Object
    +

    +
    • +
+
    +
  • Direct Relationships
    • +
  • + {% for relationship in dict_object['nb_relation'] %} +
    +
    + {{ relationship }} +
    +
    + {{ dict_object['nb_relation'][relationship] }} +
    +
    + {% endfor %} +
    • +
+ +
+
+
+
+ +
+
+ + {% include 'correlation/legend_graph_correlation.html' %} + +
+
+ + + +
+
+

Tags All Objects

+
+
+
+ + + + + + + + {% include 'tags/block_tags_selector.html' %} + +
+
+
+ +
+
+
+ + + + + + + + + +