Merge branch 'master' into misp_modules

bin/lib/Correlate_object.py

@@ -34,11 +34,25 @@ def get_all_correlation_objects():
     '''
     return ['domain', 'paste']
 
+def exist_object(object_type, correlation_id, type_id=None):
+    if object_type == 'domain':
+        return Domain.verify_if_domain_exist(correlation_id)
+    elif object_type == 'paste':
+        return Item.exist_item(correlation_id)
+    elif object_type == 'decoded':
+        return Decoded.exist_decoded(correlation_id)
+    elif object_type == 'pgp':
+        return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
+    elif object_type == 'cryptocurrency':
+        return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
+    else:
+        return False
+
 def get_object_metadata(object_type, correlation_id, type_id=None):
     if object_type == 'domain':
         return Domain.Domain(correlation_id).get_domain_metadata()
     elif object_type == 'paste':
-        return {}
+        return Item.get_item({"id": correlation_id, "date": True, "tags": True})[0]
     elif object_type == 'decoded':
         return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True)
     elif object_type == 'pgp':

bin/lib/Decoded.py

@@ -39,6 +39,9 @@ def nb_decoded_item_size(sha1_string):
     else:
         return int(nb)
 
+def exist_decoded(sha1_string):
+    return r_serv_metadata.exists('metadata_hash:{}'.format(sha1_string))
+
 def get_decoded_metadata(sha1_string, nb_seen=False, size=False):
     metadata_dict = {}
     metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen')

bin/lib/Domain.py

@@ -237,6 +237,15 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
 
     return domain_correl
 
+def get_domain_total_nb_correlation(correlation_dict):
+    total_correlation = 0
+    if 'decoded' in correlation_dict:
+        total_correlation += len(correlation_dict['decoded'])
+    if 'cryptocurrency' in correlation_dict:
+        total_correlation += correlation_dict['cryptocurrency'].get('nb', 0)
+    if 'pgp' in correlation_dict:
+        total_correlation += correlation_dict['pgp'].get('nb', 0)
+    return total_correlation
 
  # TODO: handle port
 def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem

bin/packages/Tag.py

@@ -20,6 +20,21 @@ r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
 r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 config_loader = None
 
+def build_unsafe_tags():
+    unsafe_tags = set()
+    ## CE content
+    unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
+    # add copine-scale tags
+    taxonomies = Taxonomies()
+    copine_scale = taxonomies.get('copine-scale')
+    if copine_scale:
+        for tag in copine_scale.machinetags():
+            unsafe_tags.add(tag)
+    return unsafe_tags
+
+# set of unsafe tags
+unsafe_tags = build_unsafe_tags()
+
 def get_taxonomie_from_tag(tag):
     return tag.split(':')[0]
 
@@ -46,6 +61,27 @@ def is_galaxy_tag_enabled(galaxy, tag):
     else:
         return False
 
+def enable_taxonomy(taxonomie, enable_tags=True):
+    '''
+    Enable a taxonomy. (UI)
+
+    :param taxonomie: MISP taxonomy
+    :type taxonomie: str
+    :param enable_tags: crawled domain
+    :type enable_tags: boolean
+    '''
+    taxonomies = Taxonomies()
+    if enable_tags:
+        taxonomie_info = taxonomies.get(taxonomie)
+        if taxonomie_info:
+            # activate taxonomie
+            r_serv_tags.sadd('active_taxonomies', taxonomie)
+            # activate taxonomie tags
+            for tag in taxonomie_info.machinetags():
+                r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
+        else:
+            print('Error: {}, please update pytaxonomies'.format(taxonomie))
+
 # Check if tags are enabled in AIL
 def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
     if list_tags:
@@ -74,6 +110,17 @@ def get_tag_metadata(tag):
     last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
     return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen}
 
+def is_tags_safe(ltags):
+    '''
+    Check if a list of tags contain an unsafe tag (CE, ...)
+
+    :param ltags: list of tags
+    :type ltags: list
+    :return: is a tag in the unsafe set
+    :rtype: boolean
+    '''
+    return unsafe_tags.isdisjoint(ltags)
+
 def is_tag_in_all_tag(tag):
     if r_serv_tags.sismember('list_tags', tag):
         return True

var/www/Flask_server.py

@@ -18,7 +18,13 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log
 import flask
 import importlib
 from os.path import join
+
+# # TODO: put me in lib/Tag
+from pytaxonomies import Taxonomies
+
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Tag
+
 sys.path.append('./modules/')
 
 from User import User
@@ -26,9 +32,6 @@ from User import User
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
 
-
-from pytaxonomies import Taxonomies
-
 # Import config
 import Flask_config
 
@@ -217,20 +220,15 @@ def page_not_found(e):
     return render_template('error/404.html'), 404
 
 # ========== INITIAL taxonomies ============
-# add default ail taxonomies
-r_serv_tags.sadd('active_taxonomies', 'infoleak')
-r_serv_tags.sadd('active_taxonomies', 'gdpr')
-r_serv_tags.sadd('active_taxonomies', 'fpf')
-# add default tags
-taxonomies = Taxonomies()
-for tag in taxonomies.get('infoleak').machinetags():
-    r_serv_tags.sadd('active_tag_infoleak', tag)
-for tag in taxonomies.get('gdpr').machinetags():
-    r_serv_tags.sadd('active_tag_gdpr', tag)
-for tag in taxonomies.get('fpf').machinetags():
-    r_serv_tags.sadd('active_tag_fpf', tag)
+default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
+
+# enable default taxonomies
+for taxo in default_taxonomies:
+    Tag.enable_taxonomy(taxo)
 
 # ========== INITIAL tags auto export ============
+taxonomies = Taxonomies()
+
 infoleak_tags = taxonomies.get('infoleak').machinetags()
 infoleak_automatic_tags = []
 for tag in taxonomies.get('infoleak').machinetags():

var/www/blueprints/correlation.py

@@ -10,7 +10,7 @@ import sys
 import json
 import random
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort
 from flask_login import login_required, current_user, login_user, logout_user
 
 sys.path.append('modules')
@@ -22,6 +22,7 @@ from Role_Manager import login_admin, login_analyst
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Correlate_object
+import Domain
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Cryptocurrency
@@ -108,9 +109,10 @@ def get_card_metadata(object_type, correlation_id, type_id=None):
         card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id)
         card_dict["vt"]["status"] = vt_enabled
     elif object_type == 'domain':
-        pass
+        card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
+        card_dict["tags"] = Domain.get_domain_tags(correlation_id)
     elif object_type == 'paste':
-        pass
+        card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
     return card_dict
 
 # ============= ROUTES ==============
@@ -169,6 +171,11 @@ def show_correlation():
         correlation_names = sanitise_correlation_names(request.args.get('correlation_names'))
         correlation_objects = sanitise_correlation_objects(request.args.get('correlation_objects'))
 
+        # check if correlation_id exist
+        if not Correlate_object.exist_object(object_type, correlation_id, type_id=type_id):
+            abort(404) # return 404
+        # oject exist
+        else:
             dict_object = {"object_type": object_type, "correlation_id": correlation_id}
             dict_object["max_nodes"] = max_nodes
             dict_object["mode"] = mode
@@ -180,7 +187,8 @@ def show_correlation():
             if type_id:
                 dict_object["metadata"]['type_id'] = type_id
             dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id)
-        return render_template("show_correlation.html", dict_object=dict_object)
+            return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
+
 
 @correlation.route('/correlation/graph_node_json')
 @login_required

var/www/blueprints/crawler_splash.py

@@ -21,7 +21,7 @@ from Role_Manager import create_user_db, check_password_strength, check_user_rol
 from Role_Manager import login_admin, login_analyst
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-from Tag import get_modal_add_tags
+import Tag
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Domain
@@ -63,11 +63,13 @@ def showDomain():
     dict_domain['domain'] = domain_name
     if domain.is_domain_up():
         dict_domain = {**dict_domain, **domain.get_domain_correlation()}
+        dict_domain['correlation_nb'] = Domain.get_domain_total_nb_correlation(dict_domain)
         dict_domain['origin_item'] = domain.get_domain_last_origin()
         dict_domain['tags'] = domain.get_domain_tags()
+        dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags'])
         dict_domain['history'] = domain.get_domain_history_with_status()
         dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
         dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
 
     return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
-                                modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
+                                modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain"))

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -346,6 +346,13 @@
           {% endif %}
         </div>
 
+        <div>
+					<a href="{{ url_for('correlation.show_correlation')}}?object_type=paste&correlation_id={{ request.args.get('paste') }}&correlation_objects=paste" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fa fa-search"></i> Show Paste Correlation
+						</button>
+					</a>
+				</div>
+
       </div>
 
     <div class="panel-body" id="panel-body">

var/www/templates/correlation/metadata_card_domain.html

@@ -0,0 +1,68 @@
+<div class="card my-3">
+	<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+		<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
+		<ul class="list-group mb-2">
+			<li class="list-group-item py-0">
+				<table class="table">
+					<thead>
+						<tr>
+							<th>Object type</th>
+							<th>type</th>
+							<th>First seen</th>
+							<th>Last check</th>
+							<th>Port</th>
+							<th>Status</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr>
+							<td>{{ dict_object["object_type"] }}</td>
+							<td>
+								<svg height="26" width="26">
+									<g class="nodes">
+										<circle cx="13" cy="13" r="13" fill="orange"></circle>
+										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+									</g>
+								</svg>
+								{{ dict_object["metadata"]["type_id"] }}
+							</td>
+							<td>{{ dict_object["metadata"]['first_seen'] }}</td>
+							<td>{{ dict_object["metadata"]['last_check'] }}</td>
+							<td>{{ dict_object["metadata"]['ports'] }}</td>
+							<td>
+								{% if dict_object["metadata"]["status"] %}
+									<div style="color:Green;">
+										<i class="fas fa-check-circle fa-2x"></i>
+										UP
+									</div>
+								{% else %}
+									<div style="color:Red;">
+										<i class="fas fa-times-circle fa-2x"></i>
+										DOWN
+									</div>
+								{% endif %}
+							</td>
+						</tr>
+					</tbody>
+				</table>
+			</li>
+
+			<li class="list-group-item py-0">
+				<br>
+				<div class="mb-3">
+					Tags:
+					{% for tag in dict_object["metadata_card"]['tags'] %}
+						<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+					{%  endfor %}
+				</div>
+				<div class="mb-2 float-right">
+					<a href="{{ url_for('crawler_splash.showDomain')}}?domain={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain
+						</button>
+					</a>
+				</div>
+
+			</li>
+		</ul>
+	</div>
+</div>

var/www/templates/correlation/metadata_card_paste.html

@@ -0,0 +1,50 @@
+<div class="card my-3">
+	<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+		<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
+		<ul class="list-group mb-2">
+			<li class="list-group-item py-0">
+				<table class="table">
+					<thead>
+						<tr>
+							<th>Object type</th>
+							<th>type</th>
+							<th>date</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr>
+							<td>{{ dict_object["object_type"] }}</td>
+							<td>
+								<svg height="26" width="26">
+									<g class="nodes">
+										<circle cx="13" cy="13" r="13" fill="orange"></circle>
+										<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
+									</g>
+								</svg>
+								{{ dict_object["metadata"]["type_id"] }}
+							</td>
+							<td>{{ dict_object["metadata"]['date'][0:4] }}/{{ dict_object["metadata"]['date'][4:6] }}/{{ dict_object["metadata"]['date'][6:8] }}</td>
+						</tr>
+					</tbody>
+				</table>
+			</li>
+
+			<li class="list-group-item py-0">
+				<br>
+				<div class="mb-3">
+					Tags:
+					{% for tag in dict_object["metadata"]['tags'] %}
+						<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+					{%  endfor %}
+				</div>
+				<div class="mb-2 float-right">
+					<a href="{{ url_for('showsavedpastes.showsavedpaste')}}?paste={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
+						<button class="btn btn-info"><i class="fas fa-search"></i> Show Paste
+						</button>
+					</a>
+				</div>
+
+			</li>
+		</ul>
+	</div>
+</div>

var/www/templates/correlation/show_correlation.html

@@ -94,6 +94,10 @@
 						{% include 'correlation/metadata_card_pgp.html' %}
 					{% elif dict_object["object_type"] == "decoded" %}
 						{% include 'correlation/metadata_card_decoded.html' %}
+					{% elif dict_object["object_type"] == "domain" %}
+						{% include 'correlation/metadata_card_domain.html' %}
+					{% elif dict_object["object_type"] == "paste" %}
+						{% include 'correlation/metadata_card_paste.html' %}
 					{% endif %}
 
 					<div class="row">

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -98,6 +98,17 @@
 								Last Origin: <a class="badge" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=dict_domain['origin_item']) }}" />{{ dict_domain['origin_item'] }}</a>
 								{%endif%}
 
+                {% if dict_domain['correlation_nb'] > 0 %}
+                  <hr>
+                  <div class="mt-2">
+          					<a href="{{ url_for('correlation.show_correlation')}}?object_type=domain&correlation_id={{ dict_domain['domain'] }}&correlation_objects=domain" target="_blank" style="font-size: 15px">
+          						<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain Correlations &nbsp;
+                        <div class="badge badge-warning">{{dict_domain['correlation_nb']}}</div>
+          						</button>
+          					</a>
+          				</div>
+                {%endif%}
+
 							</div>
 						</div>
 
@@ -243,7 +254,7 @@
                                     <i class="{{ var_icon }}"></i>
                                     &nbsp;&nbsp;{{ dict_key }}
                                   </td>
-                                  <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=pgp&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
+                                  <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
                                 </tr>
                               {% endfor %}
                             {% endif %}
@@ -356,12 +367,18 @@
 						<div class="card my-2" style="background-color:#ecf0f1;">
 							<div class="card-body py-2">
 								<div class="row">
-									<div class="col-md-8">
-										<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="13">
+                  <div class="col-md-3 text-center">
+										<button class="btn btn-primary" onclick="blocks.value=0;pixelate();">
+												<i class="fas fa-eye-slash"></i>
+												<span class="label-icon">Hide</span>
+										</button>
 									</div>
-									<div class="col-md-4">
+									<div class="col-md-6">
+										<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="{%if dict_domain['tags_safe']%}13{%else%}0{%endif%}">
+									</div>
+									<div class="col-md-3 text-center">
 										<button class="btn btn-primary" onclick="blocks.value=50;pixelate();">
-												<i class="fas fa-search-plu"></i>
+												<i class="fas fa-search-"></i>
 												<span class="label-icon">Full resolution</span>
 										</button>
 									</div>