chg: [UI Domain] UI: tag domain

This commit is contained in:
Terrtia 2019-11-05 09:49:51 +01:00
parent a3e5e44c9b
commit 3c6e424ac3
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
6 changed files with 282 additions and 27 deletions

View file

@ -131,7 +131,28 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F
def get_link_tree(): def get_link_tree():
pass pass
def get_domain_last_check(domain, domain_type=None, r_format="str"):
'''
Get domain last check date
:param domain: crawled domain
:type domain: str
:param domain_type: domain type
:type domain_type: str
:return: domain last check date
:rtype: str
'''
if not domain_type:
domain_type = get_domain_type(domain)
last_check = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'last_check')
if last_check is not None:
if r_format=="int":
last_check = int(last_check)
# str
else:
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
return last_check
def get_domain_tags(domain): def get_domain_tags(domain):
''' '''
@ -257,10 +278,7 @@ class Domain(object):
:return: domain last check date :return: domain last check date
:rtype: str :rtype: str
''' '''
last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check') return get_domain_last_check(self.domain, domain_type=self.type)
if last_check is not None:
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
return last_check
def is_domain_up(self): # # TODO: handle multiple ports def is_domain_up(self): # # TODO: handle multiple ports
''' '''

View file

@ -10,6 +10,7 @@ import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
import Domain
from pytaxonomies import Taxonomies from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters from pymispgalaxies import Galaxies, Clusters
@ -110,7 +111,7 @@ def get_item_tags_minimal(item_id):
return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ] return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
# TEMPLATE + API QUERY # TEMPLATE + API QUERY
def add_items_tag(tags=[], galaxy_tags=[], item_id=None): def add_items_tag(tags=[], galaxy_tags=[], item_id=None): ## TODO: remove me
res_dict = {} res_dict = {}
if item_id == None: if item_id == None:
return ({'status': 'error', 'reason': 'Item id not found'}, 404) return ({'status': 'error', 'reason': 'Item id not found'}, 404)
@ -138,18 +139,58 @@ def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
return (res_dict, 200) return (res_dict, 200)
def add_item_tag(tag, item_path): # TEMPLATE + API QUERY
def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
res_dict = {}
if item_id == None:
return ({'status': 'error', 'reason': 'Item id not found'}, 404)
if not tags and not galaxy_tags:
return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
if item_type not in ('paste', 'domain'):
return ({'status': 'error', 'reason': 'Incorrect item_type'}, 400)
item_date = int(Item.get_item_date(item_path)) res_dict['tags'] = []
for tag in tags:
if tag:
taxonomie = get_taxonomie_from_tag(tag)
if is_taxonomie_tag_enabled(taxonomie, tag):
add_item_tag(tag, item_id, item_type=item_type)
res_dict['tags'].append(tag)
else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
#add tag for tag in galaxy_tags:
r_serv_metadata.sadd('tag:{}'.format(item_path), tag) if tag:
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path) galaxy = get_galaxy_from_tag(tag)
if is_galaxy_tag_enabled(galaxy, tag):
add_item_tag(tag, item_id, item_type=item_type)
res_dict['tags'].append(tag)
else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
if Item.is_crawled(item_path): res_dict['id'] = item_id
domain = Item.get_item_domain(item_path) res_dict['type'] = item_type
r_serv_metadata.sadd('tag:{}'.format(domain), tag) return (res_dict, 200)
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
def add_item_tag(tag, item_path, item_type="paste"):
if item_type=="paste":
item_date = int(Item.get_item_date(item_path))
#add tag
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
if Item.is_crawled(item_path):
domain = Item.get_item_domain(item_path)
r_serv_metadata.sadd('tag:{}'.format(domain), tag)
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
# domain item
else:
item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path)
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1) r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
@ -250,3 +291,12 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
else: else:
tag_last_seen = Date.date_substract_day(tag_last_seen) tag_last_seen = Date.date_substract_day(tag_last_seen)
update_tag_last_seen(tag, tag_first_seen, tag_last_seen) update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
# used by modal
def get_modal_add_tags(item_id, tag_type='paste'):
'''
Modal: add tags to domain or Paste
'''
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
"item_id": item_id, "type": tag_type}

View file

@ -20,6 +20,9 @@ import Flask_config
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
from Role_Manager import login_admin, login_analyst from Role_Manager import login_admin, login_analyst
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from Tag import get_modal_add_tags
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Domain import Domain
@ -40,8 +43,8 @@ def api_validator(api_response):
if api_response: if api_response:
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1] return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
# ============= ROUTES ============== # ============= ROUTES ==============
# add route : /crawlers/show_domain
@crawler_splash.route('/crawlers/showDomain') @crawler_splash.route('/crawlers/showDomain')
#@login_required #@login_required
#@login_analyst #@login_analyst
@ -65,4 +68,5 @@ def showDomain():
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label) return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))

View file

@ -442,6 +442,28 @@ def addTags():
# success # success
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
@Tags.route("/Tags/add_item_tags")
@login_required
@login_analyst
def add_item_tags():
tags = request.args.get('tags')
tagsgalaxies = request.args.get('tagsgalaxies')
item_id = request.args.get('item_id')
item_type = request.args.get('type')
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
res = Tag.add_items_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, item_id=item_id, item_type=item_type)
# error
if res[1] != 200:
return str(res[0])
# success
if item_type=='domain':
return redirect(url_for('crawler_splash.showDomain', domain=item_id))
else:
return redirect(url_for('showsavedpastes.showsavedpaste', paste=item_id))
@Tags.route("/Tags/taxonomies") @Tags.route("/Tags/taxonomies")
@login_required @login_required

View file

@ -7,11 +7,24 @@
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS --> <!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script> <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<style>
.icon_img:hover{
cursor: pointer;
color: #17a2b8;
}
.icon_selected{
color: #007bff;
}
</style>
</head> </head>
<body> <body>
@ -77,6 +90,10 @@
</a> </a>
{% endfor %} {% endfor %}
<br> <br>
{% include 'modals/add_tags.html' %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div> </div>
</div> </div>
@ -243,7 +260,7 @@
</td> </td>
<td class="text-center"> <td class="text-center">
{%if item["screenshot"]%} {%if item["screenshot"]%}
<button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');"> <span class="icon_img" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');" id="{{ item["screenshot"].replace('/', '') }}">
<i class="far fa-image"></i> <i class="far fa-image"></i>
</button> </button>
{%endif%} {%endif%}
@ -358,8 +375,8 @@ function toggle_sidebar(){
<script> <script>
var ctx = canvas.getContext('2d'), img = new Image(); var ctx = canvas.getContext('2d'), img = new Image();
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}" var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}";
var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste=" var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste=";
/// turn off image smoothing /// turn off image smoothing
ctx.webkitImageSmoothingEnabled = false; ctx.webkitImageSmoothingEnabled = false;
@ -369,12 +386,17 @@ img.onload = pixelate;
img.addEventListener("error", img_error); img.addEventListener("error", img_error);
var draw_img = false; var draw_img = false;
{%if 'crawler_history' in dict_domain%} {%if dict_domain['crawler_history']['random_item']['screenshot']%}
var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}" var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}";
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}") var selected_icon = $("#"+screenshot.replace(/\//g, ""));
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}") selected_icon.addClass("icon_selected");
selected_icon.removeClass("icon_img");
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}");
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}");
{%else%} {%else%}
var screenshot = "" var screenshot = "";
{%endif%} {%endif%}
img.src = base_url + screenshot; img.src = base_url + screenshot;
@ -411,14 +433,22 @@ function img_error() {
pixelate; pixelate;
} }
function reload_image(screenshot, link, item_id) { function reload_image(new_screenshot, link, item_id) {
$("#"+screenshot.replace(/\//g, "")).removeClass("icon_selected").addClass("icon_img");
screenshot = new_screenshot;
img.src=base_url + screenshot; img.src=base_url + screenshot;
$("#screenshot_link").attr("href", screenshot_href + item_id) selected_icon = $("#"+screenshot.replace(/\//g, ""));
$("#screenshot_link").text(link) selected_icon.addClass("icon_selected");
selected_icon.removeClass("icon_img")
$("#screenshot_link").attr("href", screenshot_href + item_id);
$("#screenshot_link").text(link);
pixelate; pixelate;
} }
</script> </script>
</html> </html>

View file

@ -0,0 +1,131 @@
<div id="add_tags_modal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<div id="add_tags_modal_content" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
<h4>Add New Tags</h4>
</div>
<div class="modal-body">
<div class="input-group" >
<input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px">
</div>
<div class="dropdown">
<button type="button" class="btn btn-info dropdown-toggle mt-1 mb-3" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-taxonomie">
Taxonomie Selected
</button>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-taxonomie"> <!-- TODO: make dropdown-scrollable -->
<h6 class="dropdown-header">Taxonomie Tags</h6>
<button class="dropdown-item" type="button" id="all-tags-taxonomies">All Tags <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div>
{% for taxo in modal_add_tags['active_taxonomies'] %}
<button class="dropdown-item" type="button" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
{% endfor %}
</div>
</div>
<div class="input-group">
<input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px">
</div>
<div class="dropdown">
<button type="button" class="btn btn-info dropdown-toggle mt-1" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-galaxy">
Galaxy Selected
</button>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-galaxy"> <!-- TODO: make dropdown-scrollable -->
<h6 class="dropdown-header">Galaxy Tags</h6>
<button class="dropdown-item" type="button" id="all-tags-galaxies">All Tags <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div>
{% for galaxy in modal_add_tags['active_galaxies'] %}
<button class="dropdown-item" type="button" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
{% endfor %}
</div>
</div>
</div>
<div class="modal-footer">
<a class="btn btn-link" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
<span class="label-icon">Edit Taxonomies List </span>
<i class="fas fa-wrench fa-2x"></i>
</a>
<a class="btn btn-link" href="{{ url_for('Tags.galaxies') }}" target="_blank">
<span class="label-icon">Edit Galaxies List</span>
<i class="fas fa-rocket fa-2x"></i>
</a>
<button class="btn btn-primary" onclick="addTags()">
<i class="fas fa-plus"></i>
Add Tags
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
<script> // TODO: add tags to PASTE or DOMAIN
var ltags;
var ltagsgalaxies;
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags.setData(data)
});
});
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% for taxo in modal_add_tags['active_taxonomies'] %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
function(data) {
ltags.setData(data)
});
});
{% endfor %}
{% for galaxy in modal_add_tags['active_galaxies'] %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
function addTags() {
var tags = ltags.getValue()
var tagsgalaxy = ltagsgalaxies.getValue()
window.location.replace("{{ url_for('Tags.add_item_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&item_id={{ modal_add_tags['item_id'] }}&type={{ modal_add_tags['type'] }}");
}
</script>