chg: [UI Domain] UI: tag domain

bin/lib/Domain.py

@@ -131,7 +131,28 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F
 def get_link_tree():
     pass
 
+def get_domain_last_check(domain, domain_type=None, r_format="str"):
+    '''
+    Get domain last check date
 
+    :param domain: crawled domain
+    :type domain: str
+    :param domain_type: domain type
+    :type domain_type: str
+
+    :return: domain last check date
+    :rtype: str
+    '''
+    if not domain_type:
+        domain_type = get_domain_type(domain)
+    last_check = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'last_check')
+    if last_check is not None:
+        if r_format=="int":
+            last_check = int(last_check)
+        # str
+        else:
+            last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
+    return last_check
 
 def get_domain_tags(domain):
     '''
@@ -257,10 +278,7 @@ class Domain(object):
         :return: domain last check date
         :rtype: str
         '''
-        last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check')
+        return get_domain_last_check(self.domain, domain_type=self.type)
-        if last_check is not None:
-            last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
-        return last_check
 
     def is_domain_up(self): # # TODO: handle multiple ports
         '''

bin/packages/Tag.py

@@ -10,6 +10,7 @@ import Item
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
+import Domain
 
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
@@ -110,7 +111,7 @@ def get_item_tags_minimal(item_id):
     return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
 
 # TEMPLATE + API QUERY
-def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
+def add_items_tag(tags=[], galaxy_tags=[], item_id=None): ## TODO: remove me
     res_dict = {}
     if item_id == None:
         return ({'status': 'error', 'reason': 'Item id not found'}, 404)
@@ -138,18 +139,58 @@ def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
     return (res_dict, 200)
 
 
-def add_item_tag(tag, item_path):
+# TEMPLATE + API QUERY
+def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
+    res_dict = {}
+    if item_id == None:
+        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
+    if not tags and not galaxy_tags:
+        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+    if item_type not in ('paste', 'domain'):
+        return ({'status': 'error', 'reason': 'Incorrect item_type'}, 400)
 
-    item_date = int(Item.get_item_date(item_path))
+    res_dict['tags'] = []
+    for tag in tags:
+        if tag:
+            taxonomie = get_taxonomie_from_tag(tag)
+            if is_taxonomie_tag_enabled(taxonomie, tag):
+                add_item_tag(tag, item_id, item_type=item_type)
+                res_dict['tags'].append(tag)
+            else:
+                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
 
-    #add tag
+    for tag in galaxy_tags:
-    r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
+        if tag:
-    r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
+            galaxy = get_galaxy_from_tag(tag)
+            if is_galaxy_tag_enabled(galaxy, tag):
+                add_item_tag(tag, item_id, item_type=item_type)
+                res_dict['tags'].append(tag)
+            else:
+                return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
 
-    if Item.is_crawled(item_path):
+    res_dict['id'] = item_id
-        domain = Item.get_item_domain(item_path)
+    res_dict['type'] = item_type
-        r_serv_metadata.sadd('tag:{}'.format(domain), tag)
+    return (res_dict, 200)
-        r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
+
+
+def add_item_tag(tag, item_path, item_type="paste"):
+
+    if item_type=="paste":
+        item_date = int(Item.get_item_date(item_path))
+
+        #add tag
+        r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
+        r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
+
+        if Item.is_crawled(item_path):
+            domain = Item.get_item_domain(item_path)
+            r_serv_metadata.sadd('tag:{}'.format(domain), tag)
+            r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
+    # domain item
+    else:
+        item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
+        r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
+        r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path)
 
     r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
 
@@ -250,3 +291,12 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
         else:
             tag_last_seen = Date.date_substract_day(tag_last_seen)
             update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
+
+
+# used by modal
+def get_modal_add_tags(item_id, tag_type='paste'):
+    '''
+    Modal: add tags to domain or Paste
+    '''
+    return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
+            "item_id": item_id, "type": tag_type}

var/www/blueprints/crawler_splash.py

@@ -20,6 +20,9 @@ import Flask_config
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
 from Role_Manager import login_admin, login_analyst
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+from Tag import get_modal_add_tags
+
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Domain
 
@@ -40,8 +43,8 @@ def api_validator(api_response):
     if api_response:
         return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
 
-
 # ============= ROUTES ==============
+# add route : /crawlers/show_domain
 @crawler_splash.route('/crawlers/showDomain')
 #@login_required
 #@login_analyst
@@ -65,4 +68,5 @@ def showDomain():
         dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
         dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
 
-    return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label)
+    return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
+                                modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))

var/www/modules/Tags/Flask_Tags.py

@@ -442,6 +442,28 @@ def addTags():
     # success
     return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
+@Tags.route("/Tags/add_item_tags")
+@login_required
+@login_analyst
+def add_item_tags():
+
+    tags = request.args.get('tags')
+    tagsgalaxies = request.args.get('tagsgalaxies')
+    item_id = request.args.get('item_id')
+    item_type = request.args.get('type')
+
+    list_tag = tags.split(',')
+    list_tag_galaxies = tagsgalaxies.split(',')
+
+    res = Tag.add_items_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, item_id=item_id, item_type=item_type)
+    # error
+    if res[1] != 200:
+        return str(res[0])
+    # success
+    if item_type=='domain':
+        return redirect(url_for('crawler_splash.showDomain', domain=item_id))
+    else:
+        return redirect(url_for('showsavedpastes.showsavedpaste', paste=item_id))
 
 @Tags.route("/Tags/taxonomies")
 @login_required

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -7,11 +7,24 @@
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
 	<!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
   <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+  <style>
+    .icon_img:hover{
+  		cursor: pointer;
+  		color: #17a2b8;
+  	}
+    .icon_selected{
+  		color: #007bff;
+  	}
+  </style>
 
 </head>
 <body>
@@ -77,6 +90,10 @@
 									</a>
 									{%  endfor %}
 									<br>
+                  {% include 'modals/add_tags.html' %}
+                  <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+                    <i class="far fa-plus-square"></i>
+                  </button>
 								</div>
 
 							</div>
@@ -243,7 +260,7 @@
 			              	</td>
                     <td class="text-center">
                       {%if item["screenshot"]%}
-                      <button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');">
+                      <span class="icon_img" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');" id="{{ item["screenshot"].replace('/', '') }}">
                         <i class="far fa-image"></i>
                       </button>
                       {%endif%}
@@ -358,8 +375,8 @@ function toggle_sidebar(){
 
 <script>
 var ctx = canvas.getContext('2d'), img = new Image();
-var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"
+var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}";
-var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste="
+var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste=";
 
 /// turn off image smoothing
 ctx.webkitImageSmoothingEnabled = false;
@@ -369,12 +386,17 @@ img.onload = pixelate;
 img.addEventListener("error", img_error);
 var draw_img = false;
 
-{%if 'crawler_history' in dict_domain%}
+{%if dict_domain['crawler_history']['random_item']['screenshot']%}
-  var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"
+  var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}";
-  $("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}")
+  var selected_icon = $("#"+screenshot.replace(/\//g, ""));
-  $("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}")
+  selected_icon.addClass("icon_selected");
+  selected_icon.removeClass("icon_img");
+
+
+  $("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}");
+  $("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}");
 {%else%}
-var screenshot = ""
+var screenshot = "";
 {%endif%}
 
 img.src = base_url + screenshot;
@@ -411,14 +433,22 @@ function img_error() {
 	pixelate;
 }
 
-function reload_image(screenshot, link, item_id) {
+function reload_image(new_screenshot, link, item_id) {
+  $("#"+screenshot.replace(/\//g, "")).removeClass("icon_selected").addClass("icon_img");
+  screenshot = new_screenshot;
+
   img.src=base_url + screenshot;
-  $("#screenshot_link").attr("href", screenshot_href + item_id)
+  selected_icon = $("#"+screenshot.replace(/\//g, ""));
-  $("#screenshot_link").text(link)
+  selected_icon.addClass("icon_selected");
+  selected_icon.removeClass("icon_img")
+
+  $("#screenshot_link").attr("href", screenshot_href + item_id);
+  $("#screenshot_link").text(link);
   pixelate;
 }
 
 
 </script>
 
+
 </html>

var/www/templates/modals/add_tags.html

@@ -0,0 +1,131 @@
+<div id="add_tags_modal" class="modal fade" role="dialog">
+  <div class="modal-dialog modal-lg">
+
+    <div id="add_tags_modal_content" class="modal-content">
+      <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
+        <h4>Add New Tags</h4>
+      </div>
+
+      <div class="modal-body">
+
+        <div class="input-group" >
+          <input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px">
+        </div>
+
+        <div class="dropdown">
+          <button type="button" class="btn btn-info dropdown-toggle mt-1 mb-3" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-taxonomie">
+             Taxonomie Selected
+          </button>
+          <div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-taxonomie"> <!-- TODO: make dropdown-scrollable -->
+            <h6 class="dropdown-header">Taxonomie Tags</h6>
+            <button class="dropdown-item" type="button" id="all-tags-taxonomies">All Tags <i class="fas fa-tags"></i></button>
+            <div class="dropdown-divider"></div>
+            {% for taxo in modal_add_tags['active_taxonomies'] %}
+                <button class="dropdown-item" type="button" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
+            {% endfor %}
+          </div>
+        </div>
+
+        <div class="input-group">
+          <input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px">
+        </div>
+
+        <div class="dropdown">
+          <button type="button" class="btn btn-info dropdown-toggle mt-1" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-galaxy">
+             Galaxy Selected
+          </button>
+          <div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-galaxy"> <!-- TODO: make dropdown-scrollable -->
+            <h6 class="dropdown-header">Galaxy Tags</h6>
+            <button class="dropdown-item" type="button" id="all-tags-galaxies">All Tags <i class="fas fa-tags"></i></button>
+            <div class="dropdown-divider"></div>
+            {% for galaxy in modal_add_tags['active_galaxies'] %}
+                <button class="dropdown-item" type="button" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
+            {% endfor %}
+          </div>
+        </div>
+
+      </div>
+
+      <div class="modal-footer">
+        <a class="btn btn-link" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
+            <span class="label-icon">Edit Taxonomies List </span>
+            <i class="fas fa-wrench fa-2x"></i>
+        </a>
+        <a class="btn btn-link" href="{{ url_for('Tags.galaxies') }}" target="_blank">
+            <span class="label-icon">Edit Galaxies List</span>
+            <i class="fas fa-rocket fa-2x"></i>
+        </a>
+        <button class="btn btn-primary" onclick="addTags()">
+            <i class="fas fa-plus"></i>
+            Add Tags
+        </button>
+        <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+      </div>
+    </div>
+  </div>
+</div>
+
+<script> // TODO: add tags to PASTE or DOMAIN
+var ltags;
+var ltagsgalaxies;
+
+$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
+    function(data) {
+
+      ltags = $('#ltags').tagSuggest({
+        data: data,
+        maxDropHeight: 200,
+        name: 'ltags'
+      });
+    });
+
+$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
+  function(data) {
+
+    ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
+      data: data,
+      maxDropHeight: 200,
+      name: 'ltagsgalaxies'
+    });
+  });
+
+jQuery("#all-tags-taxonomies").click(function(e){
+        //change input tags list
+      $.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
+        function(data) {
+          ltags.setData(data)
+      });
+    });
+
+jQuery("#all-tags-galaxies").click(function(e){
+      $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
+        function(data) {
+          ltagsgalaxies.setData(data)
+      });
+    });
+
+{% for taxo in modal_add_tags['active_taxonomies'] %}
+    jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
+      $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
+        function(data) {
+          ltags.setData(data)
+  });
+});
+{% endfor %}
+
+{% for galaxy in modal_add_tags['active_galaxies'] %}
+    jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
+      $.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
+        function(data) {
+          ltagsgalaxies.setData(data)
+  });
+});
+{% endfor %}
+
+function addTags() {
+  var tags = ltags.getValue()
+  var tagsgalaxy = ltagsgalaxies.getValue()
+  window.location.replace("{{ url_for('Tags.add_item_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&item_id={{ modal_add_tags['item_id'] }}&type={{ modal_add_tags['type'] }}");
+}
+</script>