diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index afd82379..e8e8b975 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -34,11 +34,25 @@ def get_all_correlation_objects(): ''' return ['domain', 'paste'] +def exist_object(object_type, correlation_id, type_id=None): + if object_type == 'domain': + return Domain.verify_if_domain_exist(correlation_id) + elif object_type == 'paste': + return Item.exist_item(correlation_id) + elif object_type == 'decoded': + return Decoded.exist_decoded(correlation_id) + elif object_type == 'pgp': + return Pgp.pgp._exist_corelation_field(type_id, correlation_id) + elif object_type == 'cryptocurrency': + return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id) + else: + return False + def get_object_metadata(object_type, correlation_id, type_id=None): if object_type == 'domain': return Domain.Domain(correlation_id).get_domain_metadata() elif object_type == 'paste': - return {} + return Item.get_item({"id": correlation_id, "date": True, "tags": True})[0] elif object_type == 'decoded': return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True) elif object_type == 'pgp': diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index 1618b034..ce619fb5 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -39,6 +39,9 @@ def nb_decoded_item_size(sha1_string): else: return int(nb) +def exist_decoded(sha1_string): + return r_serv_metadata.exists('metadata_hash:{}'.format(sha1_string)) + def get_decoded_metadata(sha1_string, nb_seen=False, size=False): metadata_dict = {} metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen') diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 5b9310be..9c3eb5b0 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -237,6 +237,15 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): return domain_correl +def get_domain_total_nb_correlation(correlation_dict): + total_correlation = 0 + if 'decoded' in correlation_dict: + total_correlation += len(correlation_dict['decoded']) + if 'cryptocurrency' in correlation_dict: + total_correlation += correlation_dict['cryptocurrency'].get('nb', 0) + if 'pgp' in correlation_dict: + total_correlation += correlation_dict['pgp'].get('nb', 0) + return total_correlation # TODO: handle port def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 1e6c9fbc..1e311a02 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -20,6 +20,21 @@ r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None +def build_unsafe_tags(): + unsafe_tags = set() + ## CE content + unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') + # add copine-scale tags + taxonomies = Taxonomies() + copine_scale = taxonomies.get('copine-scale') + if copine_scale: + for tag in copine_scale.machinetags(): + unsafe_tags.add(tag) + return unsafe_tags + +# set of unsafe tags +unsafe_tags = build_unsafe_tags() + def get_taxonomie_from_tag(tag): return tag.split(':')[0] @@ -46,6 +61,27 @@ def is_galaxy_tag_enabled(galaxy, tag): else: return False +def enable_taxonomy(taxonomie, enable_tags=True): + ''' + Enable a taxonomy. (UI) + + :param taxonomie: MISP taxonomy + :type taxonomie: str + :param enable_tags: crawled domain + :type enable_tags: boolean + ''' + taxonomies = Taxonomies() + if enable_tags: + taxonomie_info = taxonomies.get(taxonomie) + if taxonomie_info: + # activate taxonomie + r_serv_tags.sadd('active_taxonomies', taxonomie) + # activate taxonomie tags + for tag in taxonomie_info.machinetags(): + r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag) + else: + print('Error: {}, please update pytaxonomies'.format(taxonomie)) + # Check if tags are enabled in AIL def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): if list_tags: @@ -74,6 +110,17 @@ def get_tag_metadata(tag): last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen} +def is_tags_safe(ltags): + ''' + Check if a list of tags contain an unsafe tag (CE, ...) + + :param ltags: list of tags + :type ltags: list + :return: is a tag in the unsafe set + :rtype: boolean + ''' + return unsafe_tags.isdisjoint(ltags) + def is_tag_in_all_tag(tag): if r_serv_tags.sismember('list_tags', tag): return True diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index aa6d3268..728222ca 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -18,7 +18,13 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log import flask import importlib from os.path import join + +# # TODO: put me in lib/Tag +from pytaxonomies import Taxonomies + sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Tag + sys.path.append('./modules/') from User import User @@ -26,9 +32,6 @@ from User import User sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader - -from pytaxonomies import Taxonomies - # Import config import Flask_config @@ -217,20 +220,15 @@ def page_not_found(e): return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ -# add default ail taxonomies -r_serv_tags.sadd('active_taxonomies', 'infoleak') -r_serv_tags.sadd('active_taxonomies', 'gdpr') -r_serv_tags.sadd('active_taxonomies', 'fpf') -# add default tags -taxonomies = Taxonomies() -for tag in taxonomies.get('infoleak').machinetags(): - r_serv_tags.sadd('active_tag_infoleak', tag) -for tag in taxonomies.get('gdpr').machinetags(): - r_serv_tags.sadd('active_tag_gdpr', tag) -for tag in taxonomies.get('fpf').machinetags(): - r_serv_tags.sadd('active_tag_fpf', tag) +default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] + +# enable default taxonomies +for taxo in default_taxonomies: + Tag.enable_taxonomy(taxo) # ========== INITIAL tags auto export ============ +taxonomies = Taxonomies() + infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index ca2972de..f17f3637 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -10,7 +10,7 @@ import sys import json import random -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -22,6 +22,7 @@ from Role_Manager import login_admin, login_analyst sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Correlate_object +import Domain sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Cryptocurrency @@ -108,9 +109,10 @@ def get_card_metadata(object_type, correlation_id, type_id=None): card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id) card_dict["vt"]["status"] = vt_enabled elif object_type == 'domain': - pass + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) + card_dict["tags"] = Domain.get_domain_tags(correlation_id) elif object_type == 'paste': - pass + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) return card_dict # ============= ROUTES ============== @@ -169,18 +171,24 @@ def show_correlation(): correlation_names = sanitise_correlation_names(request.args.get('correlation_names')) correlation_objects = sanitise_correlation_objects(request.args.get('correlation_objects')) - dict_object = {"object_type": object_type, "correlation_id": correlation_id} - dict_object["max_nodes"] = max_nodes - dict_object["mode"] = mode - dict_object["correlation_names"] = correlation_names - dict_object["correlation_names_str"] = ",".join(correlation_names) - dict_object["correlation_objects"] = correlation_objects - dict_object["correlation_objects_str"] = ",".join(correlation_objects) - dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) - if type_id: - dict_object["metadata"]['type_id'] = type_id - dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) - return render_template("show_correlation.html", dict_object=dict_object) + # check if correlation_id exist + if not Correlate_object.exist_object(object_type, correlation_id, type_id=type_id): + abort(404) # return 404 + # oject exist + else: + dict_object = {"object_type": object_type, "correlation_id": correlation_id} + dict_object["max_nodes"] = max_nodes + dict_object["mode"] = mode + dict_object["correlation_names"] = correlation_names + dict_object["correlation_names_str"] = ",".join(correlation_names) + dict_object["correlation_objects"] = correlation_objects + dict_object["correlation_objects_str"] = ",".join(correlation_objects) + dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) + if type_id: + dict_object["metadata"]['type_id'] = type_id + dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) + return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label) + @correlation.route('/correlation/graph_node_json') @login_required diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 2f142a9c..8ef41e42 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -21,7 +21,7 @@ from Role_Manager import create_user_db, check_password_strength, check_user_rol from Role_Manager import login_admin, login_analyst sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -from Tag import get_modal_add_tags +import Tag sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Domain @@ -63,11 +63,13 @@ def showDomain(): dict_domain['domain'] = domain_name if domain.is_domain_up(): dict_domain = {**dict_domain, **domain.get_domain_correlation()} + dict_domain['correlation_nb'] = Domain.get_domain_total_nb_correlation(dict_domain) dict_domain['origin_item'] = domain.get_domain_last_origin() dict_domain['tags'] = domain.get_domain_tags() + dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags']) dict_domain['history'] = domain.get_domain_history_with_status() dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, - modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain")) + modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain")) diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index 31d72436..623b9ea6 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -346,6 +346,13 @@ {% endif %} +
+ + + +
+
diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html new file mode 100644 index 00000000..b8d666b3 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -0,0 +1,68 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • + + + + + + + + + + + + + + + + + + + + + +
    Object typetypeFirst seenLast checkPortStatus
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_check'] }}{{ dict_object["metadata"]['ports'] }} + {% if dict_object["metadata"]["status"] %} +
    + + UP +
    + {% else %} +
    + + DOWN +
    + {% endif %} +
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata_card"]['tags'] %} + {{ tag }} + {% endfor %} +
    +
    + + + +
    + +
    • +
+
+
diff --git a/var/www/templates/correlation/metadata_card_paste.html b/var/www/templates/correlation/metadata_card_paste.html new file mode 100644 index 00000000..994351f5 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_paste.html @@ -0,0 +1,50 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • + + + + + + + + + + + + + + + +
    Object typetypedate
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['date'][0:4] }}/{{ dict_object["metadata"]['date'][4:6] }}/{{ dict_object["metadata"]['date'][6:8] }}
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + {{ tag }} + {% endfor %} +
    +
    + + + +
    + +
    • +
+
+
diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index f6e00ae3..bcf4d19a 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -94,6 +94,10 @@ {% include 'correlation/metadata_card_pgp.html' %} {% elif dict_object["object_type"] == "decoded" %} {% include 'correlation/metadata_card_decoded.html' %} + {% elif dict_object["object_type"] == "domain" %} + {% include 'correlation/metadata_card_domain.html' %} + {% elif dict_object["object_type"] == "paste" %} + {% include 'correlation/metadata_card_paste.html' %} {% endif %}
diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 279cba37..5784e8e3 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -98,6 +98,17 @@ Last Origin: {{ dict_domain['origin_item'] }} {%endif%} + {% if dict_domain['correlation_nb'] > 0 %} +
+
+ + + +
+ {%endif%} +
@@ -243,7 +254,7 @@   {{ dict_key }} - {{ key_id }} + {{ key_id }} {% endfor %} {% endif %} @@ -356,12 +367,18 @@
-
- +
+
-
+
+ +
+