From c219febd71cce0f58537d72025134cb4ce248ef5 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 21 Feb 2024 14:18:09 +0100 Subject: [PATCH] chg: [favicon] add favicons objects + correlation --- bin/lib/objects/Favicons.py | 61 +- configs/core.cfg.sample | 1 + var/www/Flask_server.py | 2 + var/www/blueprints/correlation.py | 3 + var/www/blueprints/objects_favicon.py | 90 +++ .../correlation/metadata_card_favicon.html | 172 +++++ .../correlation/show_correlation.html | 10 +- .../objects/favicon/FaviconDaterange.html | 627 ++++++++++++++++++ .../templates/sidebars/sidebar_objects.html | 6 + 9 files changed, 950 insertions(+), 22 deletions(-) create mode 100644 var/www/blueprints/objects_favicon.py create mode 100644 var/www/templates/correlation/metadata_card_favicon.html create mode 100644 var/www/templates/objects/favicon/FaviconDaterange.html diff --git a/bin/lib/objects/Favicons.py b/bin/lib/objects/Favicons.py index 4acdec31..c810864e 100755 --- a/bin/lib/objects/Favicons.py +++ b/bin/lib/objects/Favicons.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* +import base64 import mmh3 import os import sys -from flask import url_for +from io import BytesIO +from flask import url_for from pymisp import MISPObject sys.path.append(os.environ['AIL_BIN']) @@ -18,6 +20,7 @@ from lib.objects.abstract_daterange_object import AbstractDaterangeObject, Abstr config_loader = ConfigLoader() r_objects = config_loader.get_db_conn("Kvrocks_Objects") +FAVICON_FOLDER = config_loader.get_files_directory('favicons') baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -40,10 +43,6 @@ class Favicon(AbstractDaterangeObject): # # TODO: pass - def get_content(self, r_type='str'): - if r_type == 'str': - return self._get_field('content') - def get_link(self, flask_context=False): if flask_context: url = url_for('correlation.show_correlation', type=self.type, id=self.id) @@ -53,7 +52,24 @@ class Favicon(AbstractDaterangeObject): # TODO # CHANGE COLOR def get_svg_icon(self): - return {'style': 'fas', 'icon': '\uf20a', 'color': '#1E88E5', 'radius': 5} # f0c8 f45c + return {'style': 'fas', 'icon': '\uf089', 'color': '#E1F5D0', 'radius': 5} # f0c8 f45c f089 + + def get_rel_path(self): # TODO USE MUMUR HASH + rel_path = os.path.join(self.id[0:1], self.id[1:2], self.id[2:3], self.id[3:4], self.id[4:5], self.id[5:6], self.id[6:]) + return rel_path + + def get_filepath(self): + filename = os.path.join(FAVICON_FOLDER, self.get_rel_path()) + return os.path.realpath(filename) + + def get_file_content(self): + filepath = self.get_filepath() + with open(filepath, 'rb') as f: + file_content = BytesIO(f.read()) + return file_content + + def get_content(self, r_type='str'): + return self.get_file_content() def get_misp_object(self): obj_attrs = [] @@ -69,7 +85,7 @@ class Favicon(AbstractDaterangeObject): f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('favicon-mmh3', value=self.id)) - obj_attrs.append(obj.add_attribute('favicon', value=self.get_content(r_type='bytes'))) + obj_attrs.append(obj.add_attribute('favicon', value=self.get_content())) for obj_attr in obj_attrs: for tag in self.get_tags(): obj_attr.add_tag(tag) @@ -78,29 +94,32 @@ class Favicon(AbstractDaterangeObject): def get_meta(self, options=set()): meta = self._get_meta(options=options) meta['id'] = self.id + meta['img'] = self.id meta['tags'] = self.get_tags(r_list=True) if 'content' in options: meta['content'] = self.get_content() + if 'tags_safe' in options: + meta['tags_safe'] = self.is_tags_safe(meta['tags']) return meta - # def get_links(self): - # # TODO GET ALL URLS FROM CORRELATED ITEMS - - def create(self, content, _first_seen=None, _last_seen=None): - if not isinstance(content, str): - content = content.decode() - self._set_field('content', content) + def create(self, content): # TODO first seen / last seen options + filepath = self.get_filepath() + dirname = os.path.dirname(filepath) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(filepath, 'wb') as f: + f.write(content) self._create() - -def create_favicon(content, url=None): # TODO URL ???? - if isinstance(content, str): - content = content.encode() - favicon_id = mmh3.hash_bytes(content) +def create(b_content, size_limit=5000000, b64=False, force=False): + if isinstance(b_content, str): + b_content = b_content.encode() + b64 = base64.encodebytes(b_content) # newlines inserted after every 76 bytes of output + favicon_id = str(mmh3.hash(b64)) favicon = Favicon(favicon_id) if not favicon.exists(): - favicon.create(content) - + favicon.create(b_content) + return favicon class Favicons(AbstractDaterangeObjects): """ diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 892b6f64..26aac94d 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -7,6 +7,7 @@ crawled = crawled har = CRAWLED_SCREENSHOT screenshot = CRAWLED_SCREENSHOT/screenshot images = IMAGES +favicons = FAVICONS wordtrending_csv = var/www/static/csv/wordstrendingdata wordsfile = files/wordfile diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index a00f0f7e..ff06e953 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -52,6 +52,7 @@ from blueprints.objects_etag import objects_etag from blueprints.objects_hhhash import objects_hhhash from blueprints.chats_explorer import chats_explorer from blueprints.objects_image import objects_image +from blueprints.objects_favicon import objects_favicon Flask_dir = os.environ['AIL_FLASK'] @@ -111,6 +112,7 @@ app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_hhhash, url_prefix=baseUrl) app.register_blueprint(chats_explorer, url_prefix=baseUrl) app.register_blueprint(objects_image, url_prefix=baseUrl) +app.register_blueprint(objects_favicon, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 2b81d96f..9cd21b24 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -93,6 +93,9 @@ def show_correlation(): correl_option = request.form.get('EtagCheck') if correl_option: filter_types.append('etag') + correl_option = request.form.get('FaviconCheck') + if correl_option: + filter_types.append('favicon') correl_option = request.form.get('CveCheck') if correl_option: filter_types.append('cve') diff --git a/var/www/blueprints/objects_favicon.py b/var/www/blueprints/objects_favicon.py new file mode 100644 index 00000000..07d79d0c --- /dev/null +++ b/var/www/blueprints/objects_favicon.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only, no_cache + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import Favicons +from packages import Date + +# ============ BLUEPRINT ============ +objects_favicon = Blueprint('objects_favicon', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/favicon')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ +@objects_favicon.route('/favicon/') +@login_required +@login_read_only +@no_cache +def favicon(filename): + if not filename: + abort(404) + if not 9 <= len(filename) <= 11: + abort(404) + filename = filename.replace('/', '') + fav = Favicons.Favicon(filename) + return send_from_directory(Favicons.FAVICON_FOLDER, fav.get_rel_path(), as_attachment=False, mimetype='image') + + +@objects_favicon.route("/objects/favicons", methods=['GET']) +@login_required +@login_read_only +def objects_favicons(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = Favicons.Favicons().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + print(dict_objects) + + return render_template("FaviconDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + + +@objects_favicon.route("/objects/favicons/post", methods=['POST']) +@login_required +@login_read_only +def objects_favicons_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_favicon.objects_favicons', date_from=date_from, date_to=date_to, show_objects=show_objects)) + + +@objects_favicon.route("/objects/favicons/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_favicons_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(Favicons.Favicons().api_get_chart_nb_by_daterange(date_from, date_to)) + +# ============= ROUTES ============== + diff --git a/var/www/templates/correlation/metadata_card_favicon.html b/var/www/templates/correlation/metadata_card_favicon.html new file mode 100644 index 00000000..6b12ab3b --- /dev/null +++ b/var/www/templates/correlation/metadata_card_favicon.html @@ -0,0 +1,172 @@ + + + +{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + +
+
+

{{ dict_object["correlation_id"] }}

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    Object typeFirst seenLast seenNb seen
    + + + + {{ dict_object["metadata_card"]["icon"]["icon"] }} + + + {{ dict_object["object_type"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='favicon', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 4005e13c..04c4ee0b 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -4,7 +4,7 @@ - AIL - framework + Correlation - AIL @@ -123,6 +123,8 @@ {% include 'correlation/metadata_card_hhhash.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} + {% elif dict_object["object_type"] == "favicon" %} + {% include 'correlation/metadata_card_favicon.html' %} {% endif %}
@@ -232,6 +234,10 @@
+
+ + +
@@ -723,6 +729,8 @@ if (d.popover) { if (data["tags_safe"]) { if (data["type"] === "screenshot") { desc = desc + " + + + + Favicons - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ + +
+ + +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
+ + +
+ +
+
+
+ +
+
+
+
+
+
+ + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from[0:4] }}-{{ date_from[4:6] }}-{{ date_from[6:8] }} Favicons:

+ {% else %} +

{{ date_from[0:4] }}-{{ date_from[4:6] }}-{{ date_from[6:8] }} to {{ date_to[0:4] }}-{{ date_to[4:6] }}-{{ date_to[6:8] }} Favicons:

+ {% endif %} + +
+ {% include 'objects/image/block_blur_img_slider.html' %} +
+ + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
First SeenLast SeenTotalLast days
+ + + {{ dict_objects[obj_id]['id'] }} + + + {% if dict_objects[obj_id]['first_seen'] %} + {{ dict_objects[obj_id]['first_seen'][0:4] }}/{{ dict_objects[obj_id]['first_seen'][4:6] }}/{{ dict_objects[obj_id]['first_seen'][6:8] }} + {% else %} + {{ dict_objects[obj_id]['first_seen'] }} + {% endif %} + + {% if dict_objects[obj_id]['last_seen'] %} + {{ dict_objects[obj_id]['last_seen'][0:4] }}/{{ dict_objects[obj_id]['last_seen'][4:6] }}/{{ dict_objects[obj_id]['last_seen'][6:8] }} + {% else %} + {{ dict_objects[obj_id]['last_seen'] }} + {% endif %} + {{ dict_objects[obj_id]['nb_seen'] }}
+ + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Favicon

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Favicon

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index b6c2effb..ddba8474 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -70,6 +70,12 @@ HHHash +
  • + + + Favicon + +