From 39be4ef916eeb1f2fccb136a27fa5ef0aeb5a763 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 18 Nov 2019 09:46:15 +0100 Subject: [PATCH 1/6] chg: [Correlation UI + default taxonomies] add dark-web to default taxonomies + fix domain correlation url --- bin/packages/Tag.py | 21 ++++++++++++++ var/www/Flask_server.py | 28 +++++++++---------- .../crawler/crawler_splash/showDomain.html | 2 +- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 1e6c9fbc..08265ad8 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -46,6 +46,27 @@ def is_galaxy_tag_enabled(galaxy, tag): else: return False +def enable_taxonomy(taxonomie, enable_tags=True): + ''' + Enable a taxonomy. (UI) + + :param taxonomie: MISP taxonomy + :type taxonomie: str + :param enable_tags: crawled domain + :type enable_tags: boolean + ''' + taxonomies = Taxonomies() + if enable_tags: + taxonomie_info = taxonomies.get(taxonomie) + if taxonomie_info: + # activate taxonomie + r_serv_tags.sadd('active_taxonomies', taxonomie) + # activate taxonomie tags + for tag in taxonomie_info.machinetags(): + r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag) + else: + print('Error: {}, please update pytaxonomies'.format(taxonomie)) + # Check if tags are enabled in AIL def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): if list_tags: diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index aa6d3268..728222ca 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -18,7 +18,13 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log import flask import importlib from os.path import join + +# # TODO: put me in lib/Tag +from pytaxonomies import Taxonomies + sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Tag + sys.path.append('./modules/') from User import User @@ -26,9 +32,6 @@ from User import User sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader - -from pytaxonomies import Taxonomies - # Import config import Flask_config @@ -217,20 +220,15 @@ def page_not_found(e): return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ -# add default ail taxonomies -r_serv_tags.sadd('active_taxonomies', 'infoleak') -r_serv_tags.sadd('active_taxonomies', 'gdpr') -r_serv_tags.sadd('active_taxonomies', 'fpf') -# add default tags -taxonomies = Taxonomies() -for tag in taxonomies.get('infoleak').machinetags(): - r_serv_tags.sadd('active_tag_infoleak', tag) -for tag in taxonomies.get('gdpr').machinetags(): - r_serv_tags.sadd('active_tag_gdpr', tag) -for tag in taxonomies.get('fpf').machinetags(): - r_serv_tags.sadd('active_tag_fpf', tag) +default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] + +# enable default taxonomies +for taxo in default_taxonomies: + Tag.enable_taxonomy(taxo) # ========== INITIAL tags auto export ============ +taxonomies = Taxonomies() + infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 279cba37..16161fe9 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -243,7 +243,7 @@   {{ dict_key }} - {{ key_id }} + {{ key_id }} {% endfor %} {% endif %} From 7d442646c2c0d4d03358f0eade0d1ebdb25cb352 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 18 Nov 2019 16:35:40 +0100 Subject: [PATCH 2/6] chg: [Correlation UI] add metadata card and item link: paste + domain --- bin/lib/Correlate_object.py | 2 +- var/www/blueprints/correlation.py | 8 +- .../showpaste/templates/show_saved_paste.html | 7 + .../correlation/metadata_card_domain.html | 164 ++++++++++++++++++ .../correlation/metadata_card_paste.html | 146 ++++++++++++++++ .../correlation/show_correlation.html | 4 + .../crawler/crawler_splash/showDomain.html | 10 ++ 7 files changed, 337 insertions(+), 4 deletions(-) create mode 100644 var/www/templates/correlation/metadata_card_domain.html create mode 100644 var/www/templates/correlation/metadata_card_paste.html diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index afd82379..16b29d8f 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -38,7 +38,7 @@ def get_object_metadata(object_type, correlation_id, type_id=None): if object_type == 'domain': return Domain.Domain(correlation_id).get_domain_metadata() elif object_type == 'paste': - return {} + return Item.get_item({"id": correlation_id, "date": True, "tags": True})[0] elif object_type == 'decoded': return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True) elif object_type == 'pgp': diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index ca2972de..819e6f3f 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -22,6 +22,7 @@ from Role_Manager import login_admin, login_analyst sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Correlate_object +import Domain sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Cryptocurrency @@ -108,9 +109,10 @@ def get_card_metadata(object_type, correlation_id, type_id=None): card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id) card_dict["vt"]["status"] = vt_enabled elif object_type == 'domain': - pass + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) + card_dict["tags"] = Domain.get_domain_tags(correlation_id) elif object_type == 'paste': - pass + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) return card_dict # ============= ROUTES ============== @@ -180,7 +182,7 @@ def show_correlation(): if type_id: dict_object["metadata"]['type_id'] = type_id dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) - return render_template("show_correlation.html", dict_object=dict_object) + return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label) @correlation.route('/correlation/graph_node_json') @login_required diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index 31d72436..623b9ea6 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -346,6 +346,13 @@ {% endif %} +
+ + + +
+
diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html new file mode 100644 index 00000000..5cacd513 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -0,0 +1,164 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • + + + + + + + + + + + + + + + + + + + + + +
    Object typetypeFirst seenLast checkPortStatus
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_check'] }}{{ dict_object["metadata"]['ports'] }} + {% if dict_object["metadata"]["status"] %} +
    + + UP +
    + {% else %} +
    + + DOWN +
    + {% endif %} +
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata_card"]['tags'] %} + {{ tag }} + {% endfor %} +
    +
    + + + +
    + +
    • +
+
+
+ + + + + diff --git a/var/www/templates/correlation/metadata_card_paste.html b/var/www/templates/correlation/metadata_card_paste.html new file mode 100644 index 00000000..6a9946ea --- /dev/null +++ b/var/www/templates/correlation/metadata_card_paste.html @@ -0,0 +1,146 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • + + + + + + + + + + + + + + + +
    Object typetypedate
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['date'][0:4] }}/{{ dict_object["metadata"]['date'][4:6] }}/{{ dict_object["metadata"]['date'][6:8] }}
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + {{ tag }} + {% endfor %} +
    +
    + + + +
    + +
    • +
+
+
+ + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index f6e00ae3..bcf4d19a 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -94,6 +94,10 @@ {% include 'correlation/metadata_card_pgp.html' %} {% elif dict_object["object_type"] == "decoded" %} {% include 'correlation/metadata_card_decoded.html' %} + {% elif dict_object["object_type"] == "domain" %} + {% include 'correlation/metadata_card_domain.html' %} + {% elif dict_object["object_type"] == "paste" %} + {% include 'correlation/metadata_card_paste.html' %} {% endif %}
diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 16161fe9..75bc88b6 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -98,6 +98,16 @@ Last Origin: {{ dict_domain['origin_item'] }} {%endif%} + {% if 'decoded' in dict_domain or 'pgp' in dict_domain or 'cryptocurrency' in dict_domain%} +
+
+ + + +
+ {%endif%} +
From 2cdec205c7a7e45c7105adbdb0f899beb9648a26 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 18 Nov 2019 16:38:14 +0100 Subject: [PATCH 3/6] fix: [correlation UI] clean script --- .../correlation/metadata_card_domain.html | 96 ------------------- .../correlation/metadata_card_paste.html | 96 ------------------- 2 files changed, 192 deletions(-) diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html index 5cacd513..b8d666b3 100644 --- a/var/www/templates/correlation/metadata_card_domain.html +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -66,99 +66,3 @@ - - - - - diff --git a/var/www/templates/correlation/metadata_card_paste.html b/var/www/templates/correlation/metadata_card_paste.html index 6a9946ea..994351f5 100644 --- a/var/www/templates/correlation/metadata_card_paste.html +++ b/var/www/templates/correlation/metadata_card_paste.html @@ -48,99 +48,3 @@ - - - - - From ae9cd1d3558724d015b7e57c983244f2c2d10cb1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 19 Nov 2019 09:02:25 +0100 Subject: [PATCH 4/6] chg: [Corrrelation UI] add 404 return --- bin/lib/Correlate_object.py | 14 ++++++++++++++ bin/lib/Decoded.py | 3 +++ var/www/blueprints/correlation.py | 32 ++++++++++++++++++------------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 16b29d8f..e8e8b975 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -34,6 +34,20 @@ def get_all_correlation_objects(): ''' return ['domain', 'paste'] +def exist_object(object_type, correlation_id, type_id=None): + if object_type == 'domain': + return Domain.verify_if_domain_exist(correlation_id) + elif object_type == 'paste': + return Item.exist_item(correlation_id) + elif object_type == 'decoded': + return Decoded.exist_decoded(correlation_id) + elif object_type == 'pgp': + return Pgp.pgp._exist_corelation_field(type_id, correlation_id) + elif object_type == 'cryptocurrency': + return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id) + else: + return False + def get_object_metadata(object_type, correlation_id, type_id=None): if object_type == 'domain': return Domain.Domain(correlation_id).get_domain_metadata() diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index 1618b034..ce619fb5 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -39,6 +39,9 @@ def nb_decoded_item_size(sha1_string): else: return int(nb) +def exist_decoded(sha1_string): + return r_serv_metadata.exists('metadata_hash:{}'.format(sha1_string)) + def get_decoded_metadata(sha1_string, nb_seen=False, size=False): metadata_dict = {} metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen') diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 819e6f3f..f17f3637 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -10,7 +10,7 @@ import sys import json import random -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -171,18 +171,24 @@ def show_correlation(): correlation_names = sanitise_correlation_names(request.args.get('correlation_names')) correlation_objects = sanitise_correlation_objects(request.args.get('correlation_objects')) - dict_object = {"object_type": object_type, "correlation_id": correlation_id} - dict_object["max_nodes"] = max_nodes - dict_object["mode"] = mode - dict_object["correlation_names"] = correlation_names - dict_object["correlation_names_str"] = ",".join(correlation_names) - dict_object["correlation_objects"] = correlation_objects - dict_object["correlation_objects_str"] = ",".join(correlation_objects) - dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) - if type_id: - dict_object["metadata"]['type_id'] = type_id - dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) - return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label) + # check if correlation_id exist + if not Correlate_object.exist_object(object_type, correlation_id, type_id=type_id): + abort(404) # return 404 + # oject exist + else: + dict_object = {"object_type": object_type, "correlation_id": correlation_id} + dict_object["max_nodes"] = max_nodes + dict_object["mode"] = mode + dict_object["correlation_names"] = correlation_names + dict_object["correlation_names_str"] = ",".join(correlation_names) + dict_object["correlation_objects"] = correlation_objects + dict_object["correlation_objects_str"] = ",".join(correlation_objects) + dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) + if type_id: + dict_object["metadata"]['type_id'] = type_id + dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) + return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label) + @correlation.route('/correlation/graph_node_json') @login_required From 15bd1cc78f523eb7598596cc7b69ff6cee757698 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 19 Nov 2019 14:03:23 +0100 Subject: [PATCH 5/6] chg: [Domain UI] add unsafe_tags_check (taxo: dark-web + copine-scale) + hide unsafe domain img --- bin/packages/Tag.py | 26 +++++++++++++++++++ var/www/blueprints/crawler_splash.py | 5 ++-- .../crawler/crawler_splash/showDomain.html | 14 +++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 08265ad8..1e311a02 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -20,6 +20,21 @@ r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None +def build_unsafe_tags(): + unsafe_tags = set() + ## CE content + unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') + # add copine-scale tags + taxonomies = Taxonomies() + copine_scale = taxonomies.get('copine-scale') + if copine_scale: + for tag in copine_scale.machinetags(): + unsafe_tags.add(tag) + return unsafe_tags + +# set of unsafe tags +unsafe_tags = build_unsafe_tags() + def get_taxonomie_from_tag(tag): return tag.split(':')[0] @@ -95,6 +110,17 @@ def get_tag_metadata(tag): last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen} +def is_tags_safe(ltags): + ''' + Check if a list of tags contain an unsafe tag (CE, ...) + + :param ltags: list of tags + :type ltags: list + :return: is a tag in the unsafe set + :rtype: boolean + ''' + return unsafe_tags.isdisjoint(ltags) + def is_tag_in_all_tag(tag): if r_serv_tags.sismember('list_tags', tag): return True diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 2f142a9c..3b8c68b1 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -21,7 +21,7 @@ from Role_Manager import create_user_db, check_password_strength, check_user_rol from Role_Manager import login_admin, login_analyst sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -from Tag import get_modal_add_tags +import Tag sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Domain @@ -65,9 +65,10 @@ def showDomain(): dict_domain = {**dict_domain, **domain.get_domain_correlation()} dict_domain['origin_item'] = domain.get_domain_last_origin() dict_domain['tags'] = domain.get_domain_tags() + dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags']) dict_domain['history'] = domain.get_domain_history_with_status() dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, - modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain")) + modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain")) diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 75bc88b6..90e87fee 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -366,12 +366,18 @@
-
- +
+
-
+
+ +
+
From 73afdfa227b096f611aefa2b24e2fabbcaaa4578 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 19 Nov 2019 14:30:12 +0100 Subject: [PATCH 6/6] chg: [Domain UI] add total_nb_correlation --- bin/lib/Domain.py | 9 +++++++++ var/www/blueprints/crawler_splash.py | 1 + var/www/templates/crawler/crawler_splash/showDomain.html | 5 +++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 5b9310be..9c3eb5b0 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -237,6 +237,15 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): return domain_correl +def get_domain_total_nb_correlation(correlation_dict): + total_correlation = 0 + if 'decoded' in correlation_dict: + total_correlation += len(correlation_dict['decoded']) + if 'cryptocurrency' in correlation_dict: + total_correlation += correlation_dict['cryptocurrency'].get('nb', 0) + if 'pgp' in correlation_dict: + total_correlation += correlation_dict['pgp'].get('nb', 0) + return total_correlation # TODO: handle port def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 3b8c68b1..8ef41e42 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -63,6 +63,7 @@ def showDomain(): dict_domain['domain'] = domain_name if domain.is_domain_up(): dict_domain = {**dict_domain, **domain.get_domain_correlation()} + dict_domain['correlation_nb'] = Domain.get_domain_total_nb_correlation(dict_domain) dict_domain['origin_item'] = domain.get_domain_last_origin() dict_domain['tags'] = domain.get_domain_tags() dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags']) diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 90e87fee..5784e8e3 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -98,11 +98,12 @@ Last Origin: {{ dict_domain['origin_item'] }} {%endif%} - {% if 'decoded' in dict_domain or 'pgp' in dict_domain or 'cryptocurrency' in dict_domain%} + {% if dict_domain['correlation_nb'] > 0 %}
-