chg: [core + UI] search domain by tags

This commit is contained in:
Terrtia 2020-01-10 16:52:55 +01:00
parent b4a36d812f
commit 99897ffa9b
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
7 changed files with 439 additions and 3 deletions

View file

@ -23,6 +23,12 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def is_valid_object_type(object_type):
if object_type in ['domain', 'item', 'screenshot']:
return True
else:
return False
def get_all_objects():
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
@ -189,6 +195,13 @@ def get_item_url(correlation_name, value, correlation_type=None):
url = url_for(endpoint, paste=value)
return url
def get_obj_tag_table_keys(object_type):
'''
Warning: use only in flask (dynamic templates)
'''
if object_type=="domain":
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
def create_graph_links(links_set):
graph_links_list = []
@ -316,6 +329,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
######## API EXPOSED ########
def sanitize_object_type(object_type):
if not is_valid_object_type(object_type):
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
######## ########

View file

@ -200,6 +200,20 @@ def get_tag_metadata(tag, r_int=False):
tag_metadata['last_seen'] = get_tag_last_seen(tag)
return tag_metadata
def get_tags_min_last_seen(l_tags, r_int=False):
'''
Get max last seen from a list of tags (current: item only)
'''
min_last_seen = 99999999
for tag in l_tags:
last_seen = get_tag_last_seen(tag, r_int=True)
if last_seen < min_last_seen:
min_last_seen = last_seen
if r_int:
return min_last_seen
else:
return str(min_last_seen)
def is_obj_tagged(object_id, tag):
'''
Check if a object is tagged
@ -431,6 +445,106 @@ def delete_obj_tags(object_id, object_type, tags=[]):
if res:
return res
def sanitise_tags_date_range(l_tags, date_from=None, date_to=None):
if date_from and date_to is None:
date_from = get_tags_min_last_seen(l_tags, r_int=False)
date_to = date_from
return Date.sanitise_date_range(date_from, date_to)
# # TODO: verify tags + object_type
# get set_keys: intersection
def get_obj_keys_by_tags(object_type, l_tags, date_day=None):
l_set_keys = []
if object_type=='item':
for tag in l_tags:
l_set_keys.append('{}:{}'.format(tag, date_day))
else:
for tag in l_tags:
l_set_keys.append('{}:{}'.format(object_type, tag))
return l_set_keys
def get_obj_by_tag(key_tag):
return r_serv_tags.smembers(key_tag)
def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object
# with daterange
l_tagged_obj = []
if object_type=='item':
#sanityze date
date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to)
l_dates = Date.substract_date(date_from, date_to)
for date_day in l_dates:
l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day)
# if len(l_set_keys) > nb_obj:
# return l_tagged_obj
if len(l_set_keys) < 2:
date_day_obj = get_obj_by_tag(l_set_keys[0])
else:
date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
# next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj
# if next_nb_start > 0:
# get + filter nb_start
l_tagged_obj.extend( date_day_obj )
# handle pagination
nb_pages = len(l_tagged_obj) / nb_obj
if not nb_pages.is_integer():
nb_pages = int(nb_pages)+1
else:
nb_pages = int(nb_pages)
if page > nb_pages:
page = nb_pages
# select index
start = nb_obj*(page -1)
stop = (nb_obj*page) -1
l_tagged_obj = l_tagged_obj[start:stop]
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
# without daterange
else:
l_set_keys = get_obj_keys_by_tags(object_type, l_tags)
if len(l_set_keys) < 2:
l_tagged_obj = get_obj_by_tag(l_set_keys[0])
else:
l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
if not l_tagged_obj:
return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
# handle pagination
nb_pages = len(l_tagged_obj) / nb_obj
if not nb_pages.is_integer():
nb_pages = int(nb_pages)+1
else:
nb_pages = int(nb_pages)
if page > nb_pages:
page = nb_pages
# multiple pages
if nb_pages > 1:
start = nb_obj*(page -1)
stop = (nb_obj*page) -1
current_index = 0
l_obj = []
for elem in l_tagged_obj:
if current_index > stop:
break
if start <= current_index and stop >= current_index:
l_obj.append(elem)
current_index += 1
l_tagged_obj = l_obj
# only one page
else:
l_tagged_obj = list(l_tagged_obj)
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
def get_obj_date(object_type, object_id): # # TODO: move me in another file
if object_type == "item":
return Item.get_item_date(object_id)

View file

@ -18,7 +18,7 @@ import Flask_config
# Import Role_Manager
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
from Role_Manager import login_admin, login_analyst
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Date
@ -77,6 +77,64 @@ def delete_tag():
return str(res[0])
return redirect(Correlate_object.get_item_url(object_type, object_id))
@tags_ui.route('/tag/get_all_tags')
@login_required
@login_read_only
def get_all_tags():
return jsonify(Tag.get_all_tags())
@tags_ui.route('/tag/get_all_obj_tags')
@login_required
@login_read_only
def get_all_obj_tags():
object_type = request.args.get('object_type')
res = Correlate_object.sanitize_object_type(object_type)
if res:
return jsonify(res)
return jsonify(Tag.get_all_obj_tags(object_type))
@tags_ui.route('/tag/search/get_obj_by_tags')
@login_required
@login_read_only
def get_obj_by_tags():
# # TODO: sanityze all
object_type = request.args.get('object_type')
ltags = request.args.get('ltags')
page = request.args.get('ltags')
date_from = request.args.get('ltags')
date_to = request.args.get('ltags')
# unpack tags
list_tags = ltags.split(',')
list_tag = []
for tag in list_tags:
list_tag.append(tag.replace('"','\"'))
res = Correlate_object.sanitize_object_type(object_type)
if res:
return jsonify(res)
dict_obj = Tag.get_obj_by_tags(object_type, list_tag)
if dict_obj['tagged_obj']:
dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]}
for obj_id in dict_obj['tagged_obj']:
obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
obj_metadata['id'] = obj_id
dict_tagged["tagged_obj"].append(obj_metadata)
dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
if len(list_tag) == 1:
dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '')
else:
dict_tagged['current_tags'] = list_tag
#return jsonify(dict_tagged)
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
# # add route : /crawlers/show_domain
# @tags_ui.route('/tags/search/domain')
# @login_required

View file

@ -253,6 +253,7 @@ def dashboard():
statDomains_regular = get_stats_last_crawled_domains('regular', date)
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
object_type='domain',
crawler_enabled=crawler_enabled, date=date,
crawler_metadata_regular=crawler_metadata_regular,
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)

View file

@ -105,6 +105,8 @@
</div>
</div>
{% include 'tags/block_obj_tags_search.html' %}
</div>
</div>
</div>

View file

@ -0,0 +1,101 @@
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title">Search Domain by Tags :</h5>
</div>
<div class="card-body">
<!-- <div class="row mb-3"> TODO: use condition
<div class="col-md-6">
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
</div>
</div>
<div class="col-md-6">
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
</div>
</div>
</div> -->
<div class="input-group mb-3">
<div class="input-group-prepend">
<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
<i class="fas fa-eraser"></i>
</button>
</div>
<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
</div>
<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
<i class="fas fa-search"></i> Search Domains
</button>
</div>
</div>
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<script>
var ltags;
$.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{dict_tagged['object_type']}}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
value: [{%if "current_tags" in dict_tagged%}{% for tag in dict_tagged['current_tags'] %}'{{tag|safe}}',{%endfor%}{%endif%}],
sortOrder: 'name',
maxDropHeight: 200,
name: 'ltags'
});
});
function searchTags() {
var data = ltags.getValue();
//var date_from = $('#date-range-from-input').val();
//var date_to =$('#date-range-to-input').val();
//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}?ltags=" + data + "&object_type={{dict_tagged['object_type']}}&page={{dict_tagged['page']}}");
}
function emptyTags() {
ltags.clear();
}
// $('#date-range-from').dateRangePicker({
// separator : ' to ',
// getValue: function(){
// if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
// return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
// else
// return '';
// },
// setValue: function(s,s1,s2){
// $('#date-range-from-input').val(s1);
// $('#date-range-to-input').val(s2);
// }
// });
// $('#date-range-to').dateRangePicker({
// separator : ' to ',
// getValue: function(){
// if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
// return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
// else
// return '';
// },
// setValue: function(s,s1,s2){
// $('#date-range-from-input').val(s1);
// $('#date-range-to-input').val(s2);
// }
// });
// tagid + objtype + objid
// $('#edit_tags_modal').on('show.bs.modal', function (event) {
// var button = $(event.relatedTarget);
// var tagid = button.data('tagid')
// var objtype = button.data('objtype')
// var objid = button.data('objid')
// var modal = $(this)
// modal.find('#modal_tag_edit_tag_id').text(tagid)
// modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
// })
</script>

View file

@ -0,0 +1,146 @@
<!DOCTYPE html>
<html>
<head>
<title>Tags - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="850px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% include 'tags/block_obj_tags_search.html' %}
<div>
{%if dict_tagged%}
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
{%if dict_tagged["object_type"]=="domain"%}
<th>first seen</th>
<th>last check</th>
<th style="max-width: 800px;">Domain</th>
<th>status</th>
{%endif%}
</tr>
</thead>
<tbody>
{%if dict_tagged["object_type"]=="domain"%}
{% for dict_obj in dict_tagged["tagged_obj"] %}
<tr>
<td class="pb-0">{{ dict_obj['first_seen'] }}</td>
<td class="pb-0">{{ dict_obj['last_check'] }}</td>
<td class="pb-0">
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{dict_obj['id']}}" class="text-secondary">
<div style="line-height:0.9;">{{ dict_obj['id'] }}</div>
</a>
<div class="mb-2">
{% for tag in dict_obj['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
</div>
</td>
<td class="pb-0">
{%if dict_obj['status'] %}
<div style="color:Green;">
<i class="fas fa-check-circle"></i>
UP
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle"></i>
DOWN
</div>
{% endif %}
</td>
</tr>
{% endfor %}
{% endif %}
</tbody>
</table>
{%endif%}
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var search_table;
$(document).ready(function(){
//search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
$('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
});
</script>
<script>
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>