From fb320ca5d5eab1597d9205c84aa96fb6d494f2f3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 13 Jan 2020 11:10:03 +0100 Subject: [PATCH] chg: [UI tags] search domains by tags --- bin/lib/Correlate_object.py | 2 +- bin/packages/Tag.py | 15 ++++-- var/www/blueprints/tags_ui.py | 39 +++++++++++++-- var/www/modules/Tags/templates/Tags.html | 4 +- .../hiddenServices/Flask_hiddenServices.py | 1 - .../templates/Crawler_dashboard.html | 4 +- var/www/templates/pagination.html | 50 +++++++++++++++++++ .../templates/tags/block_obj_tags_search.html | 12 +++-- var/www/templates/tags/menu_sidebar.html | 23 +++++++++ .../templates/tags/search_obj_by_tags.html | 40 ++++++--------- 10 files changed, 147 insertions(+), 43 deletions(-) create mode 100644 var/www/templates/pagination.html diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 68fbed9e..32cb382a 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -24,7 +24,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None def is_valid_object_type(object_type): - if object_type in ['domain', 'item', 'screenshot']: + if object_type in ['domain', 'item', 'image']: return True else: return False diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 45c657da..4e86c2aa 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -490,7 +490,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 l_tagged_obj.extend( date_day_obj ) # handle pagination - nb_pages = len(l_tagged_obj) / nb_obj + nb_all_elem = len(l_tagged_obj) + nb_pages = nb_all_elem / nb_obj if not nb_pages.is_integer(): nb_pages = int(nb_pages)+1 else: @@ -503,7 +504,7 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 stop = (nb_obj*page) -1 l_tagged_obj = l_tagged_obj[start:stop] - return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages} + return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop+1, "nb_all_elem":nb_all_elem} # without daterange else: @@ -517,7 +518,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0} # handle pagination - nb_pages = len(l_tagged_obj) / nb_obj + nb_all_elem = len(l_tagged_obj) + nb_pages = nb_all_elem / nb_obj if not nb_pages.is_integer(): nb_pages = int(nb_pages)+1 else: @@ -538,11 +540,16 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 l_obj.append(elem) current_index += 1 l_tagged_obj = l_obj + stop += 1 + if stop > nb_all_elem: + stop = nb_all_elem # only one page else: + start = 0 + stop = nb_all_elem l_tagged_obj = list(l_tagged_obj) - return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages} + return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem} def get_obj_date(object_type, object_id): # # TODO: move me in another file diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index d39b14b1..a1220770 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -94,6 +94,22 @@ def get_all_obj_tags(): return jsonify(res) return jsonify(Tag.get_all_obj_tags(object_type)) +@tags_ui.route('/tag/search/domain') +@login_required +@login_read_only +def tags_search_domains(): + object_type = 'domain' + dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + +@tags_ui.route('/tag/search/image') +@login_required +@login_read_only +def tags_search_images(): + object_type = 'image' + dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + @tags_ui.route('/tag/search/get_obj_by_tags') @login_required @login_read_only @@ -102,7 +118,7 @@ def get_obj_by_tags(): # # TODO: sanityze all object_type = request.args.get('object_type') ltags = request.args.get('ltags') - page = request.args.get('ltags') + page = request.args.get('page') date_from = request.args.get('ltags') date_to = request.args.get('ltags') @@ -112,14 +128,23 @@ def get_obj_by_tags(): for tag in list_tags: list_tag.append(tag.replace('"','\"')) + # object_type res = Correlate_object.sanitize_object_type(object_type) if res: return jsonify(res) - dict_obj = Tag.get_obj_by_tags(object_type, list_tag) + # page + try: + page = int(page) + except: + page = 1 + + dict_obj = Tag.get_obj_by_tags(object_type, list_tag, page=page) if dict_obj['tagged_obj']: - dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]} + dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s", + "tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], + "nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']} for obj_id in dict_obj['tagged_obj']: obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id) obj_metadata['id'] = obj_id @@ -128,12 +153,16 @@ def get_obj_by_tags(): dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type) if len(list_tag) == 1: - dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '') + dict_tagged['current_tags'] = [ltags.replace('"', '\"')] else: dict_tagged['current_tags'] = list_tag + dict_tagged['current_tags_str'] = ltags #return jsonify(dict_tagged) - return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + else: + dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + + return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) # # add route : /crawlers/show_domain # @tags_ui.route('/tags/search/domain') diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html index d538269c..b4fb85c8 100644 --- a/var/www/modules/Tags/templates/Tags.html +++ b/var/www/modules/Tags/templates/Tags.html @@ -217,8 +217,8 @@ var last_clicked_paste; var can_change_modal_content = true; $(document).ready(function(){ - $("#nav_quick_search").removeClass("text-muted"); - $("#nav_tag_{{tag_nav}}").addClass("active"); + $("#nav_tags_search").removeClass("text-muted"); + $("#nav_tags_search_item").addClass("active"); search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] }); // Use to bind the button with the new displayed data diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index b92374ef..0db6bbe6 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -253,7 +253,6 @@ def dashboard(): statDomains_regular = get_stats_last_crawled_domains('regular', date) return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion, - object_type='domain', crawler_enabled=crawler_enabled, date=date, crawler_metadata_regular=crawler_metadata_regular, statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular) diff --git a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html index ee67be8b..dfca6ddc 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html +++ b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html @@ -105,7 +105,9 @@ - {% include 'tags/block_obj_tags_search.html' %} + {% with object_type='domain' %} + {% include 'tags/block_obj_tags_search.html' %} + {% endwith %} diff --git a/var/www/templates/pagination.html b/var/www/templates/pagination.html new file mode 100644 index 00000000..3da2e289 --- /dev/null +++ b/var/www/templates/pagination.html @@ -0,0 +1,50 @@ +
+
+ +
+ + {%if nb_all_elem%} +
+ + {{object_name}}:  + {{nb_first_elem}}-{{nb_last_elem}} + / + {{nb_all_elem}} + +
+
+
+
+ {%endif%} +
diff --git a/var/www/templates/tags/block_obj_tags_search.html b/var/www/templates/tags/block_obj_tags_search.html index e4e34e07..8669f4db 100644 --- a/var/www/templates/tags/block_obj_tags_search.html +++ b/var/www/templates/tags/block_obj_tags_search.html @@ -1,6 +1,6 @@
-
Search Domain by Tags :
+
Search {{object_name}} by Tags :
@@ -29,21 +29,22 @@
+