AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-27 00:07:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into misp_modules

Browse source
This commit is contained in:
Terrtia 2019-11-20 11:42:39 +01:00
commit f4fa811229
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
12 changed files with 265 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
bin/lib/Correlate_object.py
View file

@ -34,11 +34,25 @@ def get_all_correlation_objects():
'''
return ['domain', 'paste']
def exist_object(object_type, correlation_id, type_id=None):
if object_type == 'domain':
return Domain.verify_if_domain_exist(correlation_id)
elif object_type == 'paste':
return Item.exist_item(correlation_id)
elif object_type == 'decoded':
return Decoded.exist_decoded(correlation_id)
elif object_type == 'pgp':
return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
else:
return False
def get_object_metadata(object_type, correlation_id, type_id=None):
if object_type == 'domain':
return Domain.Domain(correlation_id).get_domain_metadata()
elif object_type == 'paste':
return {}
return Item.get_item({"id": correlation_id, "date": True, "tags": True})[0]
elif object_type == 'decoded':
return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True)
elif object_type == 'pgp':

3
bin/lib/Decoded.py
View file

@ -39,6 +39,9 @@ def nb_decoded_item_size(sha1_string):
else:
return int(nb)
def exist_decoded(sha1_string):
return r_serv_metadata.exists('metadata_hash:{}'.format(sha1_string))
def get_decoded_metadata(sha1_string, nb_seen=False, size=False):
metadata_dict = {}
metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen')

9
bin/lib/Domain.py
View file

@ -237,6 +237,15 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
return domain_correl
def get_domain_total_nb_correlation(correlation_dict):
total_correlation = 0
if 'decoded' in correlation_dict:
total_correlation += len(correlation_dict['decoded'])
if 'cryptocurrency' in correlation_dict:
total_correlation += correlation_dict['cryptocurrency'].get('nb', 0)
if 'pgp' in correlation_dict:
total_correlation += correlation_dict['pgp'].get('nb', 0)
return total_correlation
# TODO: handle port
def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem

47
bin/packages/Tag.py
View file

@ -20,6 +20,21 @@ r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def build_unsafe_tags():
unsafe_tags = set()
## CE content
unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
# add copine-scale tags
taxonomies = Taxonomies()
copine_scale = taxonomies.get('copine-scale')
if copine_scale:
for tag in copine_scale.machinetags():
unsafe_tags.add(tag)
return unsafe_tags
# set of unsafe tags
unsafe_tags = build_unsafe_tags()
def get_taxonomie_from_tag(tag):
return tag.split(':')[0]
@ -46,6 +61,27 @@ def is_galaxy_tag_enabled(galaxy, tag):
else:
return False
def enable_taxonomy(taxonomie, enable_tags=True):
'''
Enable a taxonomy. (UI)
:param taxonomie: MISP taxonomy
:type taxonomie: str
:param enable_tags: crawled domain
:type enable_tags: boolean
'''
taxonomies = Taxonomies()
if enable_tags:
taxonomie_info = taxonomies.get(taxonomie)
if taxonomie_info:
# activate taxonomie
r_serv_tags.sadd('active_taxonomies', taxonomie)
# activate taxonomie tags
for tag in taxonomie_info.machinetags():
r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
else:
print('Error: {}, please update pytaxonomies'.format(taxonomie))
# Check if tags are enabled in AIL
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
if list_tags:
@ -74,6 +110,17 @@ def get_tag_metadata(tag):
last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen}
def is_tags_safe(ltags):
'''
Check if a list of tags contain an unsafe tag (CE, ...)
:param ltags: list of tags
:type ltags: list
:return: is a tag in the unsafe set
:rtype: boolean
'''
return unsafe_tags.isdisjoint(ltags)
def is_tag_in_all_tag(tag):
if r_serv_tags.sismember('list_tags', tag):
return True

28
var/www/Flask_server.py
View file

@ -18,7 +18,13 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log
import flask
import importlib
from os.path import join
# # TODO: put me in lib/Tag
from pytaxonomies import Taxonomies
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
sys.path.append('./modules/')
from User import User
@ -26,9 +32,6 @@ from User import User
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from pytaxonomies import Taxonomies
# Import config
import Flask_config
@ -217,20 +220,15 @@ def page_not_found(e):
return render_template('error/404.html'), 404
# ========== INITIAL taxonomies ============
# add default ail taxonomies
r_serv_tags.sadd('active_taxonomies', 'infoleak')
r_serv_tags.sadd('active_taxonomies', 'gdpr')
r_serv_tags.sadd('active_taxonomies', 'fpf')
# add default tags
taxonomies = Taxonomies()
for tag in taxonomies.get('infoleak').machinetags():
r_serv_tags.sadd('active_tag_infoleak', tag)
for tag in taxonomies.get('gdpr').machinetags():
r_serv_tags.sadd('active_tag_gdpr', tag)
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
# enable default taxonomies
for taxo in default_taxonomies:
Tag.enable_taxonomy(taxo)
# ========== INITIAL tags auto export ============
taxonomies = Taxonomies()
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():

38
var/www/blueprints/correlation.py
View file

@ -10,7 +10,7 @@ import sys
import json
import random
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort
from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules')
@ -22,6 +22,7 @@ from Role_Manager import login_admin, login_analyst
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Correlate_object
import Domain
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Cryptocurrency
@ -108,9 +109,10 @@ def get_card_metadata(object_type, correlation_id, type_id=None):
card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id)
card_dict["vt"]["status"] = vt_enabled
elif object_type == 'domain':
pass
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
card_dict["tags"] = Domain.get_domain_tags(correlation_id)
elif object_type == 'paste':
pass
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
return card_dict
# ============= ROUTES ==============
@ -169,18 +171,24 @@ def show_correlation():
correlation_names = sanitise_correlation_names(request.args.get('correlation_names'))
correlation_objects = sanitise_correlation_objects(request.args.get('correlation_objects'))
dict_object = {"object_type": object_type, "correlation_id": correlation_id}
dict_object["max_nodes"] = max_nodes
dict_object["mode"] = mode
dict_object["correlation_names"] = correlation_names
dict_object["correlation_names_str"] = ",".join(correlation_names)
dict_object["correlation_objects"] = correlation_objects
dict_object["correlation_objects_str"] = ",".join(correlation_objects)
dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id)
if type_id:
dict_object["metadata"]['type_id'] = type_id
dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id)
return render_template("show_correlation.html", dict_object=dict_object)
# check if correlation_id exist
if not Correlate_object.exist_object(object_type, correlation_id, type_id=type_id):
abort(404) # return 404
# oject exist
else:
dict_object = {"object_type": object_type, "correlation_id": correlation_id}
dict_object["max_nodes"] = max_nodes
dict_object["mode"] = mode
dict_object["correlation_names"] = correlation_names
dict_object["correlation_names_str"] = ",".join(correlation_names)
dict_object["correlation_objects"] = correlation_objects
dict_object["correlation_objects_str"] = ",".join(correlation_objects)
dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id)
if type_id:
dict_object["metadata"]['type_id'] = type_id
dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id)
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
@correlation.route('/correlation/graph_node_json')
@login_required

6
var/www/blueprints/crawler_splash.py
View file

@ -21,7 +21,7 @@ from Role_Manager import create_user_db, check_password_strength, check_user_rol
from Role_Manager import login_admin, login_analyst
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from Tag import get_modal_add_tags
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Domain
@ -63,11 +63,13 @@ def showDomain():
dict_domain['domain'] = domain_name
if domain.is_domain_up():
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
dict_domain['correlation_nb'] = Domain.get_domain_total_nb_correlation(dict_domain)
dict_domain['origin_item'] = domain.get_domain_last_origin()
dict_domain['tags'] = domain.get_domain_tags()
dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags'])
dict_domain['history'] = domain.get_domain_history_with_status()
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain"))

7
var/www/modules/showpaste/templates/show_saved_paste.html
View file

@ -346,6 +346,13 @@
{% endif %}
</div>
<div>
<a href="{{ url_for('correlation.show_correlation')}}?object_type=paste&correlation_id={{ request.args.get('paste') }}&correlation_objects=paste" target="_blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fa fa-search"></i> Show Paste Correlation
</button>
</a>
</div>
</div>
<div class="panel-body" id="panel-body">

68
var/www/templates/correlation/metadata_card_domain.html Normal file
View file

@ -0,0 +1,68 @@
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>type</th>
<th>First seen</th>
<th>Last check</th>
<th>Port</th>
<th>Status</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ dict_object["object_type"] }}</td>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
</g>
</svg>
{{ dict_object["metadata"]["type_id"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_check'] }}</td>
<td>{{ dict_object["metadata"]['ports'] }}</td>
<td>
{% if dict_object["metadata"]["status"] %}
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
UP
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle fa-2x"></i>
DOWN
</div>
{% endif %}
</td>
</tr>
</tbody>
</table>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in dict_object["metadata_card"]['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</div>
<div class="mb-2 float-right">
<a href="{{ url_for('crawler_splash.showDomain')}}?domain={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain
</button>
</a>
</div>
</li>
</ul>
</div>
</div>

50
var/www/templates/correlation/metadata_card_paste.html Normal file
View file

@ -0,0 +1,50 @@
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ dict_object["correlation_id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>type</th>
<th>date</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ dict_object["object_type"] }}</td>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text>
</g>
</svg>
{{ dict_object["metadata"]["type_id"] }}
</td>
<td>{{ dict_object["metadata"]['date'][0:4] }}/{{ dict_object["metadata"]['date'][4:6] }}/{{ dict_object["metadata"]['date'][6:8] }}</td>
</tr>
</tbody>
</table>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in dict_object["metadata"]['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</div>
<div class="mb-2 float-right">
<a href="{{ url_for('showsavedpastes.showsavedpaste')}}?paste={{ dict_object["correlation_id"] }}" target="_blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fas fa-search"></i> Show Paste
</button>
</a>
</div>
</li>
</ul>
</div>
</div>

4
var/www/templates/correlation/show_correlation.html
View file

@ -94,6 +94,10 @@
{% include 'correlation/metadata_card_pgp.html' %}
{% elif dict_object["object_type"] == "decoded" %}
{% include 'correlation/metadata_card_decoded.html' %}
{% elif dict_object["object_type"] == "domain" %}
{% include 'correlation/metadata_card_domain.html' %}
{% elif dict_object["object_type"] == "paste" %}
{% include 'correlation/metadata_card_paste.html' %}
{% endif %}
<div class="row">

27
var/www/templates/crawler/crawler_splash/showDomain.html
View file

@ -98,6 +98,17 @@
Last Origin: <a class="badge" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=dict_domain['origin_item']) }}" />{{ dict_domain['origin_item'] }}</a>
{%endif%}
{% if dict_domain['correlation_nb'] > 0 %}
<hr>
<div class="mt-2">
<a href="{{ url_for('correlation.show_correlation')}}?object_type=domain&correlation_id={{ dict_domain['domain'] }}&correlation_objects=domain" target="_blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fas fa-search"></i> Show Domain Correlations &nbsp;
<div class="badge badge-warning">{{dict_domain['correlation_nb']}}</div>
</button>
</a>
</div>
{%endif%}
</div>
</div>
@ -243,7 +254,7 @@
<i class="{{ var_icon }}"></i>
&nbsp;&nbsp;{{ dict_key }}
</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=pgp&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&correlation_id={{ key_id }}&type_id={{ dict_key }}&correlation_objects=domain">{{ key_id }}</a></td>
</tr>
{% endfor %}
{% endif %}
@ -356,12 +367,18 @@
<div class="card my-2" style="background-color:#ecf0f1;">
<div class="card-body py-2">
<div class="row">
<div class="col-md-8">
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="13">
<div class="col-md-3 text-center">
<button class="btn btn-primary" onclick="blocks.value=0;pixelate();">
<i class="fas fa-eye-slash"></i>
<span class="label-icon">Hide</span>
</button>
</div>
<div class="col-md-4">
<div class="col-md-6">
<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="{%if dict_domain['tags_safe']%}13{%else%}0{%endif%}">
</div>
<div class="col-md-3 text-center">
<button class="btn btn-primary" onclick="blocks.value=50;pixelate();">
<i class="fas fa-search-plu"></i>
<i class="fas fa-search-"></i>
<span class="label-icon">Full resolution</span>
</button>
</div>