chg: [core + UI] search domain by tags

bin/lib/Correlate_object.py

@@ -23,6 +23,12 @@ config_loader = ConfigLoader.ConfigLoader()
 r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 config_loader = None
 
+def is_valid_object_type(object_type):
+    if object_type in ['domain', 'item', 'screenshot']:
+        return True
+    else:
+        return False
+
 def get_all_objects():
     return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
 
@@ -189,6 +195,13 @@ def get_item_url(correlation_name, value, correlation_type=None):
         url = url_for(endpoint, paste=value)
     return url
 
+def get_obj_tag_table_keys(object_type):
+    '''
+    Warning: use only in flask (dynamic templates)
+    '''
+    if object_type=="domain":
+        return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
+
 
 def create_graph_links(links_set):
     graph_links_list = []
@@ -316,6 +329,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
 
 
 ######## API EXPOSED ########
-
+def sanitize_object_type(object_type):
-
+    if not is_valid_object_type(object_type):
+        return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
 ########  ########

bin/packages/Tag.py

@@ -200,6 +200,20 @@ def get_tag_metadata(tag, r_int=False):
     tag_metadata['last_seen'] = get_tag_last_seen(tag)
     return tag_metadata
 
+def get_tags_min_last_seen(l_tags, r_int=False):
+    '''
+    Get max last seen from a list of tags (current: item only)
+    '''
+    min_last_seen = 99999999
+    for tag in l_tags:
+        last_seen = get_tag_last_seen(tag, r_int=True)
+        if last_seen < min_last_seen:
+                min_last_seen = last_seen
+    if r_int:
+        return min_last_seen
+    else:
+        return str(min_last_seen)
+
 def is_obj_tagged(object_id, tag):
     '''
     Check if a object is tagged
@@ -431,6 +445,106 @@ def delete_obj_tags(object_id, object_type, tags=[]):
         if res:
             return res
 
+def sanitise_tags_date_range(l_tags, date_from=None, date_to=None):
+    if date_from and date_to is None:
+        date_from = get_tags_min_last_seen(l_tags, r_int=False)
+        date_to = date_from
+    return Date.sanitise_date_range(date_from, date_to)
+
+
+# # TODO: verify tags + object_type
+# get set_keys: intersection
+def get_obj_keys_by_tags(object_type, l_tags, date_day=None):
+    l_set_keys = []
+    if object_type=='item':
+        for tag in l_tags:
+            l_set_keys.append('{}:{}'.format(tag, date_day))
+    else:
+        for tag in l_tags:
+            l_set_keys.append('{}:{}'.format(object_type, tag))
+    return l_set_keys
+
+def get_obj_by_tag(key_tag):
+    return r_serv_tags.smembers(key_tag)
+
+def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object
+    # with daterange
+    l_tagged_obj = []
+    if object_type=='item':
+        #sanityze date
+        date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to)
+        l_dates = Date.substract_date(date_from, date_to)
+
+        for date_day in l_dates:
+            l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day)
+            # if len(l_set_keys) > nb_obj:
+            #     return l_tagged_obj
+            if len(l_set_keys) < 2:
+                date_day_obj = get_obj_by_tag(l_set_keys[0])
+            else:
+                date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
+
+            # next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj
+            # if next_nb_start > 0:
+            #  get + filter nb_start
+            l_tagged_obj.extend( date_day_obj )
+
+        # handle pagination
+        nb_pages = len(l_tagged_obj) / nb_obj
+        if not nb_pages.is_integer():
+            nb_pages = int(nb_pages)+1
+        else:
+            nb_pages = int(nb_pages)
+        if page > nb_pages:
+            page = nb_pages
+
+        # select index
+        start = nb_obj*(page -1)
+        stop = (nb_obj*page) -1
+        l_tagged_obj = l_tagged_obj[start:stop]
+
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
+
+    # without daterange
+    else:
+        l_set_keys = get_obj_keys_by_tags(object_type, l_tags)
+        if len(l_set_keys) < 2:
+            l_tagged_obj = get_obj_by_tag(l_set_keys[0])
+        else:
+            l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
+
+        if not l_tagged_obj:
+            return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
+
+        # handle pagination
+        nb_pages = len(l_tagged_obj) / nb_obj
+        if not nb_pages.is_integer():
+            nb_pages = int(nb_pages)+1
+        else:
+            nb_pages = int(nb_pages)
+        if page > nb_pages:
+            page = nb_pages
+
+        # multiple pages
+        if nb_pages > 1:
+            start = nb_obj*(page -1)
+            stop = (nb_obj*page) -1
+            current_index = 0
+            l_obj = []
+            for elem in l_tagged_obj:
+                if current_index > stop:
+                    break
+                if start <= current_index and stop >= current_index:
+                    l_obj.append(elem)
+                current_index += 1
+            l_tagged_obj = l_obj
+        # only one page
+        else:
+            l_tagged_obj = list(l_tagged_obj)
+
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
+
+
 def get_obj_date(object_type, object_id): # # TODO: move me in another file
     if object_type == "item":
         return Item.get_item_date(object_id)

var/www/blueprints/tags_ui.py

@@ -18,7 +18,7 @@ import Flask_config
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Date
@@ -77,6 +77,64 @@ def delete_tag():
         return str(res[0])
     return redirect(Correlate_object.get_item_url(object_type, object_id))
 
+
+@tags_ui.route('/tag/get_all_tags')
+@login_required
+@login_read_only
+def get_all_tags():
+    return jsonify(Tag.get_all_tags())
+
+@tags_ui.route('/tag/get_all_obj_tags')
+@login_required
+@login_read_only
+def get_all_obj_tags():
+    object_type = request.args.get('object_type')
+    res = Correlate_object.sanitize_object_type(object_type)
+    if res:
+        return jsonify(res)
+    return jsonify(Tag.get_all_obj_tags(object_type))
+
+@tags_ui.route('/tag/search/get_obj_by_tags')
+@login_required
+@login_read_only
+def get_obj_by_tags():
+
+    # # TODO: sanityze all
+    object_type = request.args.get('object_type')
+    ltags = request.args.get('ltags')
+    page = request.args.get('ltags')
+    date_from = request.args.get('ltags')
+    date_to = request.args.get('ltags')
+
+    # unpack tags
+    list_tags = ltags.split(',')
+    list_tag = []
+    for tag in list_tags:
+        list_tag.append(tag.replace('"','\"'))
+
+    res = Correlate_object.sanitize_object_type(object_type)
+    if res:
+        return jsonify(res)
+
+    dict_obj = Tag.get_obj_by_tags(object_type, list_tag)
+
+    if dict_obj['tagged_obj']:
+        dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]}
+        for obj_id in dict_obj['tagged_obj']:
+            obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
+            obj_metadata['id'] = obj_id
+            dict_tagged["tagged_obj"].append(obj_metadata)
+
+        dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
+
+        if len(list_tag) == 1:
+            dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '')
+        else:
+            dict_tagged['current_tags'] = list_tag
+
+        #return jsonify(dict_tagged)
+        return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
 # # add route : /crawlers/show_domain
 # @tags_ui.route('/tags/search/domain')
 # @login_required

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -253,6 +253,7 @@ def dashboard():
     statDomains_regular = get_stats_last_crawled_domains('regular', date)
 
     return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
+                                object_type='domain',
                                 crawler_enabled=crawler_enabled, date=date,
                                 crawler_metadata_regular=crawler_metadata_regular,
                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -105,6 +105,8 @@
 					</div>
 				</div>
 
+				{% include 'tags/block_obj_tags_search.html' %}
+
 			</div>
 		</div>
 	</div>

var/www/templates/tags/block_obj_tags_search.html

@@ -0,0 +1,101 @@
+<div class="card mb-3 mt-1">
+	<div class="card-header text-white bg-dark">
+		<h5 class="card-title">Search Domain by Tags :</h5>
+	</div>
+		<div class="card-body">
+
+			<!-- <div class="row mb-3">                      TODO: use condition
+				<div class="col-md-6">
+					<div class="input-group" id="date-range-from">
+						<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+						<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
+					</div>
+				</div>
+				<div class="col-md-6">
+					<div class="input-group" id="date-range-to">
+						<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+						<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
+					</div>
+				</div>
+			</div> -->
+
+			<div class="input-group mb-3">
+			  <div class="input-group-prepend">
+					<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
+						<i class="fas fa-eraser"></i>
+					</button>
+			  </div>
+				<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
+			</div>
+
+			<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
+				<i class="fas fa-search"></i> Search Domains
+			</button>
+
+			</div>
+	</div>
+
+<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+<script>
+  var ltags;
+  $.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{dict_tagged['object_type']}}",
+    function(data) {
+      ltags = $('#ltags').tagSuggest({
+        data: data,
+				value: [{%if "current_tags" in dict_tagged%}{% for tag in dict_tagged['current_tags'] %}'{{tag|safe}}',{%endfor%}{%endif%}],
+        sortOrder: 'name',
+        maxDropHeight: 200,
+        name: 'ltags'
+      });
+  });
+
+  function searchTags() {
+  	var data = ltags.getValue();
+  	//var date_from = $('#date-range-from-input').val();
+  	//var date_to =$('#date-range-to-input').val();
+		//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
+  	window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}?ltags=" + data + "&object_type={{dict_tagged['object_type']}}&page={{dict_tagged['page']}}");
+  }
+  function emptyTags() {
+  	ltags.clear();
+  }
+
+	// $('#date-range-from').dateRangePicker({
+	// 			separator : ' to ',
+	// 			getValue: function(){
+	// 					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+	// 							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+	// 					else
+	// 							return '';
+	// 					},
+	// 					setValue: function(s,s1,s2){
+	// 							$('#date-range-from-input').val(s1);
+	// 							$('#date-range-to-input').val(s2);
+	// 					}
+	// });
+	// $('#date-range-to').dateRangePicker({
+	// 		separator : ' to ',
+	// 		getValue: function(){
+	// 				if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+	// 						return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+	// 				else
+	// 						return '';
+	// 		},
+	// 		setValue: function(s,s1,s2){
+	// 				$('#date-range-from-input').val(s1);
+	// 				$('#date-range-to-input').val(s2);
+	// 		}
+	// });
+
+// tagid + objtype + objid
+// $('#edit_tags_modal').on('show.bs.modal', function (event) {
+//     var button = $(event.relatedTarget);
+//     var tagid = button.data('tagid')
+//     var objtype = button.data('objtype')
+//     var objid = button.data('objid')
+//     var modal = $(this)
+//     modal.find('#modal_tag_edit_tag_id').text(tagid)
+//     modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
+//   })
+</script>

var/www/templates/tags/search_obj_by_tags.html

@@ -0,0 +1,146 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Tags - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<!-- Modal -->
+  <div id="mymodal" class="modal fade" role="dialog">
+   <div class="modal-dialog modal-lg">
+
+     <!-- Modal content-->
+     <div id="mymodalcontent" class="modal-content">
+       <div id="mymodalbody" class="modal-body" max-width="850px">
+         <p>Loading paste information...</p>
+         <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
+       </div>
+       <div class="modal-footer">
+         <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
+         <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+       </div>
+     </div>
+   </div>
+  </div>
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'tags/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				{% include 'tags/block_obj_tags_search.html' %}
+
+				<div>
+					{%if dict_tagged%}
+						<table class="table table-bordered table-hover" id="myTable_">
+				      <thead class="thead-dark">
+				        <tr>
+                  {%if dict_tagged["object_type"]=="domain"%}
+  				          <th>first seen</th>
+  				          <th>last check</th>
+                    <th style="max-width: 800px;">Domain</th>
+  				          <th>status</th>
+                  {%endif%}
+				        </tr>
+				      </thead>
+				      <tbody>
+
+                {%if dict_tagged["object_type"]=="domain"%}
+
+  				        {% for dict_obj in dict_tagged["tagged_obj"] %}
+  				          <tr>
+                      <td class="pb-0">{{ dict_obj['first_seen'] }}</td>
+                      <td class="pb-0">{{ dict_obj['last_check'] }}</td>
+  										<td class="pb-0">
+                        <a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{dict_obj['id']}}" class="text-secondary">
+                          <div style="line-height:0.9;">{{ dict_obj['id'] }}</div>
+                        </a>
+                        <div class="mb-2">
+  				                {% for tag in dict_obj['tags'] %}
+    				                <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
+    				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+    				                </a>
+  				                {%  endfor %}
+  				              </div>
+                      </td>
+  				            <td class="pb-0">
+                        {%if dict_obj['status'] %}
+      										<div style="color:Green;">
+      											<i class="fas fa-check-circle"></i>
+      											UP
+      										</div>
+      									{% else %}
+      										<div style="color:Red;">
+      											<i class="fas fa-times-circle"></i>
+      											DOWN
+      										</div>
+      									{% endif %}
+  										</td>
+  				          </tr>
+  				        {%  endfor %}
+
+                {%  endif %}
+
+				      </tbody>
+				    </table>
+
+					{%endif%}
+				</div>
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+var ltags;
+var search_table;
+
+$(document).ready(function(){
+	//search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
+  $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
+
+});
+</script>
+
+<script>
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</html>