chg: [correlation] tags all objects

bin/lib/correlations_engine.py

@@ -118,6 +118,33 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob
     r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}')
     r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}')
 
+# # bypass max result/objects ???
+# def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300):
+#     objs = set()
+#     _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max)
+#     return objs
+#
+# def _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max, previous_str_obj=''):
+#     obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
+#     objs.add(obj_str_id)
+#
+#     obj_correlations = get_correlations(obj_type, subtype, obj_id, filter_types=filter_types)
+#     for correl_type in obj_correlations:
+#         for str_obj in obj_correlations[correl_type]:
+#             subtype2, obj2_id = str_obj.split(':', 1)
+#             obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
+#
+#             if obj2_str_id == previous_str_obj:
+#                 continue
+#
+#             if len(nodes) > nb_max:
+#                 break
+#             objs.add(obj2_str_id)
+#
+#             if level > 0:
+#                 next_level = level - 1
+#                 _get_correlation_depht(objs, correl_type, subtype2, obj2_id, filter_types, next_level, nb_max,
+#                                        previous_str_obj=obj_str_id)
 
 def get_obj_str_id(obj_type, subtype, obj_id):
     if subtype is None:

bin/lib/objects/abstract_object.py

@@ -213,11 +213,11 @@ class AbstractObject(ABC):
         """
         return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type])
 
-    def get_correlations(self):
+    def get_correlations(self, filter_types=[]):
         """
         Get object correlations
         """
-        return get_correlations(self.type, self.subtype, self.id)
+        return get_correlations(self.type, self.subtype, self.id, filter_types=filter_types)
 
     def get_nb_correlation(self, correl_type):
         return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type)

bin/lib/objects/ail_objects.py

@@ -273,27 +273,31 @@ def get_obj_correlations(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.get_correlations()
 
-def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
+def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
-    if lvl > 0 and (obj_type, subtype, obj_id) not in objs:  # Avoid looking for the same correlation
+    if len(objs) < nb_max or nb_max == -1:
-        objs.add((obj_type, subtype, obj_id))
+        if lvl == 0:
-        lvl = lvl - 1
+            objs.add((obj_type, subtype, obj_id))
-        obj = get_object(obj_type, subtype, obj_id)
+
-        correlations = obj.get_correlations()
+        elif lvl > 0 and (obj_type, subtype, obj_id) not in objs:  # Avoid looking for the same correlation
-        # print('--------------------------')
+            objs.add((obj_type, subtype, obj_id))
-        # print( obj_id, correlations)
+            obj = get_object(obj_type, subtype, obj_id)
-        # print(lvl)
+            correlations = obj.get_correlations(filter_types=filter_types)
-        # print('--------------------------')
+            lvl = lvl - 1
-        for obj2_type in correlations:
+            for obj2_type in correlations:
-            for str_obj in correlations[obj2_type]:
+                for str_obj in correlations[obj2_type]:
-                obj2_subtype, obj2_id = str_obj.split(':', 1)
+                    obj2_subtype, obj2_id = str_obj.split(':', 1)
-                _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
+                    _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
-    # print(len(objs))
+
-    objs.add((obj_type, subtype, obj_id))
+def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
+    objs = set()
+    _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
     return objs
 
-def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
+def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
-    objs = set()
+    objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
-    _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
+    for obj_tuple in objs:
+        obj1_type, subtype1, id1 = obj_tuple
+        add_obj_tags(obj1_type, subtype1, id1, tags)
     return objs
 
 ################################################################################

var/www/blueprints/correlation.py

@@ -24,22 +24,17 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.objects import ail_objects
+from lib import Tag
 
 bootstrap_label = Flask_config.bootstrap_label
 vt_enabled = Flask_config.vt_enabled
 
 # ============ BLUEPRINT ============
-correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
+correlation = Blueprint('correlation', __name__,
+                        template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
 
 # ============ VARIABLES ============
 
-######
-### graph_line_json
-### 'hashDecoded.pgpdump_graph_line_json'
-### 'hashDecoded.cryptocurrency_graph_line_json'
-###
-######
-
 # ============ FUNCTIONS ============
 
 def sanitise_graph_mode(graph_mode):
@@ -53,12 +48,12 @@ def sanitise_nb_max_nodes(nb_max_nodes):
         nb_max_nodes = int(nb_max_nodes)
         if nb_max_nodes < 2:
             nb_max_nodes = 300
-    except:
+    except (TypeError, ValueError):
         nb_max_nodes = 300
     return nb_max_nodes
 
 # ============= ROUTES ==============
-@correlation.route('/correlation/show', methods=['GET', 'POST']) # GET + POST
+@correlation.route('/correlation/show', methods=['GET', 'POST'])
 @login_required
 @login_read_only
 def show_correlation():
@@ -106,7 +101,7 @@ def show_correlation():
 
         # redirect to keep history and bookmark
         return redirect(url_for('correlation.show_correlation', type=object_type, subtype=subtype, id=obj_id, mode=mode,
-                                            max_nodes=max_nodes, filter=filter_types))
+                                max_nodes=max_nodes, filter=filter_types))
 
     # request.method == 'GET'
     else:
@@ -120,13 +115,9 @@ def show_correlation():
 
         filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
 
-        # # TODO: remove me, rename screenshot to image
-        if obj_type == 'image':
-            obj_type = 'screenshot'
-
         # check if obj_id exist
         if not ail_objects.exists_obj(obj_type, subtype, obj_id):
-            abort(404) # return 404
+            return abort(404)
         # object exist
         else:
             dict_object = {"object_type": obj_type,
@@ -138,7 +129,8 @@ def show_correlation():
             if subtype:
                 dict_object["metadata"]['type_id'] = subtype
             dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
-            return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
+            return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
+                                   tags_selector_data=Tag.get_tags_selector_data())
 
 @correlation.route('/correlation/get/description')
 @login_required
@@ -158,7 +150,6 @@ def get_description():
     else:
         return jsonify({})
 
-
     # check if correlation_id exist
     # # TODO: return error json
     if not ail_objects.exists_obj(object_type, type_id, correlation_id):
@@ -179,11 +170,48 @@ def graph_node_json():
 
     filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
 
-    # # TODO: remove me, rename screenshot
-    if obj_type == 'image':
-        obj_type = 'screenshot'
-
     json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=2, flask_context=True)
     #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
     return jsonify(json_graph)
 
+@correlation.route('/correlation/tags/add', methods=['POST'])
+@login_required
+@login_read_only
+def correlation_tags_add():
+    obj_id = request.form.get('tag_obj_id')
+    subtype = request.form.get('tag_subtype', '')
+    obj_type = request.form.get('tag_obj_type')
+    nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
+    filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
+
+    if not ail_objects.exists_obj(obj_type, subtype, obj_id):
+        return abort(404)
+
+    # tags
+    taxonomies_tags = request.form.get('taxonomies_tags')
+    if taxonomies_tags:
+        try:
+            taxonomies_tags = json.loads(taxonomies_tags)
+        except Exception:
+            taxonomies_tags = []
+    else:
+        taxonomies_tags = []
+    galaxies_tags = request.form.get('galaxies_tags')
+    if galaxies_tags:
+        try:
+            galaxies_tags = json.loads(galaxies_tags)
+        except Exception:
+            galaxies_tags = []
+    if taxonomies_tags or galaxies_tags:
+        if not Tag.is_valid_tags_taxonomies_galaxy(taxonomies_tags, galaxies_tags):
+            return {'error': 'Invalid tag(s)'}, 400
+        tags = taxonomies_tags + galaxies_tags
+    else:
+        tags = []
+
+    if tags:
+        ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=2, nb_max=nb_max)
+
+    return redirect(url_for('correlation.show_correlation',
+                            type=obj_type, subtype=subtype, id=obj_id,
+                            filter=",".join(filter_types)))

var/www/templates/correlation/show_correlation.html

@@ -16,7 +16,7 @@
 		<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
 		<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
 		<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
-		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+		<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
 
 		<style>
 		.icon_legend {
@@ -253,7 +253,26 @@
 								</div>
 							</div>
 						</div>
-						{% endif %}
+                    {% endif %}
+
+                    <div class="card my-2">
+                        <div class="card-header bg-light">
+                            <h4><i class="fas fa-tags"></i> Tags All Objects</h4>
+                        </div>
+                        <div class="card-body">
+                            <form action="{{ url_for('correlation.correlation_tags_add') }}" method='post' onsubmit="SubmitAddTags();">
+                                <input type="hidden" id="tag_obj_type" name="tag_obj_type" value="{{ dict_object["object_type"] }}">
+                                <input type="hidden" id="tag_subtype" name="tag_subtype" value="{{ dict_object["metadata"]["type_id"] }}">
+                                <input type="hidden" id="tag_obj_id" name="tag_obj_id" value="{{ dict_object["correlation_id"] }}">
+                                <input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
+                                <input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
+                                {% include 'tags/block_tags_selector.html' %}
+                                <button class="btn btn-primary mt-2">
+									<i class="fas fa-tag"></i> Add Tags
+                                </button>
+                            </form>
+                        </div>
+                    </div>
 
 				</div>
 			</div>
@@ -292,6 +311,14 @@ function toggle_sidebar(){
 		$('#core_content').addClass('col-lg-10')
 	}
 }
+
+function SubmitAddTags() {
+	var tags = ltags.getValue();
+	var tagsgalaxy = ltagsgalaxies.getValue();
+	$('#ltags').val(tags);
+	$('#ltagsgalaxies').val(tagsgalaxy);
+	return true;
+}
 </script>
 
 <script>