chg: [chats] add image object + show message image

.gitignore

@@ -16,6 +16,7 @@ tlsh
 Blooms
 PASTES
 CRAWLED_SCREENSHOT
+IMAGES
 BASE64
 HASHS
 DATA_ARDB

bin/importer/feeders/Default.py

@@ -62,6 +62,9 @@ class DefaultFeeder:
         """
         return self.json_data.get('data')
 
+    def get_obj_type(self):
+        return self.json_data.get('type', 'item')
+
     ## OVERWRITE ME ##
     def get_obj(self):
         """

bin/importer/feeders/abstract_chats_feeder.py

@@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from importer.feeders.Default import DefaultFeeder
 from lib.objects.Chats import Chat
 from lib.objects import ChatSubChannels
+from lib.objects import Images
 from lib.objects import Messages
 from lib.objects import UsersAccount
 from lib.objects.Usernames import Username
@@ -70,7 +71,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
     def get_chat_id(self):  # TODO RAISE ERROR IF NONE
         return self.json_data['meta']['chat']['id']
 
-    def get_channel_id(self):
+    def get_subchannel_id(self):
         pass
 
     def get_subchannels(self):
@@ -114,19 +115,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         timestamp = self.get_message_timestamp()
 
         #### Create Object ID ####
-        chat_id = str(self.json_data['meta']['chat']['id'])
+        chat_id = self.get_chat_id()
-        message_id = str(self.json_data['meta']['id'])
+        message_id = self.get_message_id()
         # channel id
         # thread id
 
-        obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
+        # TODO sanitize obj type
-        self.obj = Messages.Message(obj_id)
+        obj_type = self.get_obj_type()
+        print(obj_type)
+
+        if obj_type == 'image':
+            self.obj = Images.Image(self.json_data['data-sha256'])
+
+        else:
+            obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
+            self.obj = Messages.Message(obj_id)
         return self.obj
 
-    def process_chat(self, message, date, timestamp, reply_id=None):  # TODO threads
+    def process_chat(self, obj, date, timestamp, reply_id=None):  # TODO threads
-        meta = self.json_data['meta']['chat']
+        meta = self.json_data['meta']['chat'] # todo replace me by function
         chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
-        chat.add(date)  # TODO ### Dynamic subtype ???
+
+        # date stat + correlation
+        chat.add(date, obj)
 
         if meta.get('name'):
             chat.set_name(meta['name'])
@@ -142,10 +153,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             chat.update_username_timeline(username.get_global_id(), timestamp)
 
         if meta.get('subchannel'):
-            subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id)
+            subchannel = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
             chat.add_children(obj_global_id=subchannel.get_global_id())
         else:
-            chat.add_message(message.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
+            if obj.type == 'message':
+                chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
+
 
         # if meta.get('subchannels'): # TODO Update icon + names
 
@@ -154,9 +167,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
     # def process_subchannels(self):
     #     pass
 
-    def process_subchannel(self, message, date, timestamp, reply_id=None):  # TODO CREATE DATE
+    def process_subchannel(self, obj, date, timestamp, reply_id=None):  # TODO CREATE DATE
         meta = self.json_data['meta']['chat']['subchannel']
         subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid())
+
+        # TODO correlation with obj = message/image
         subchannel.add(date)
 
         if meta.get('date'): # TODO check if already exists
@@ -169,13 +184,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         if meta.get('info'):
             subchannel.set_info(meta['info'])
 
-        subchannel.add_message(message.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
+        if obj.type == 'message':
+            subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
         return subchannel
 
-    def process_sender(self, date, timestamp):
+    def process_sender(self, obj, date, timestamp):
         meta = self.json_data['meta']['sender']
         user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
 
+        # date stat + correlation
+        user_account.add(date, obj)
+
         if meta.get('username'):
             username = Username(meta['username'], self.get_chat_protocol())
             # TODO timeline or/and correlation ????
@@ -214,25 +233,45 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
 
         # TODO Translation
 
-        # Content
+        print(self.obj.type)
-        content = self.get_message_content()
 
-        message = Messages.create(self.obj.id, content)  # TODO translation
+        # get object by meta object type
+        if self.obj.type == 'message':
+            # Content
+            obj = Messages.create(self.obj.id, self.get_message_content())  # TODO translation
 
-        # CHAT
+        else:
-        chat = self.process_chat(message, date, timestamp, reply_id=reply_id)
+            chat_id = self.get_chat_id()
+            message_id = self.get_message_id()
+            message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
+            message = Messages.Message(message_id)
+            if message.exists():
+                obj = Images.create(self.get_message_content())
+                obj.add(date, message)
+                obj.set_parent(obj_global_id=message.get_global_id())
+            else:
+                obj = None
 
-        # SENDER # TODO HANDLE NULL SENDER
+        if obj:
-        user_account = self.process_sender(date, timestamp)
 
-        # UserAccount---Message
+            # CHAT
-        user_account.add(date, obj=message)
+            chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
-        # UserAccount---Chat
-        user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
 
-        # if chat: # TODO Chat---Username correlation ???
+            # SENDER # TODO HANDLE NULL SENDER
-        #     # Chat---Username
+            user_account = self.process_sender(obj, date, timestamp)
-        #     chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+
+            # UserAccount---Chat
+            user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
+
+            # if chat: # TODO Chat---Username correlation ???
+            #     # Chat---Username    => need to handle members and participants
+            #     chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+
+
+            # TODO Sender image -> correlation
+                # image
+                #       -> subchannel ?
+                #       -> thread id ?
 
 
 

bin/lib/ail_core.py

@@ -16,8 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
 r_object = config_loader.get_db_conn("Kvrocks_Objects")
 config_loader = None
 
-AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
+AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash',
-                      'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
+                      'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
 def get_ail_uuid():
     ail_uuid = r_serv_db.get('ail:uuid')

bin/lib/correlations_engine.py

@@ -41,7 +41,7 @@ config_loader = None
 ##################################
 
 CORRELATION_TYPES_BY_OBJ = {
-    "chat": ["user-account"],  # message or direct correlation like cve, bitcoin, ... ???
+    "chat": ["image", "user-account"],  # message or direct correlation like cve, bitcoin, ... ???
     "cookie-name": ["domain"],
     "cryptocurrency": ["domain", "item", "message"],
     "cve": ["domain", "item", "message"],
@@ -50,8 +50,9 @@ CORRELATION_TYPES_BY_OBJ = {
     "etag": ["domain"],
     "favicon": ["domain", "item"],  # TODO Decoded
     "hhhash": ["domain"],
+    "image": ["chat", "message", "user-account"],
     "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],  # chat ???
-    "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"],  # chat ??
+    "message": ["cve", "cryptocurrency", "decoded", "image", "pgp", "user-account"],  # chat ??
     "pgp": ["domain", "item", "message"],
     "screenshot": ["domain", "item"],
     "title": ["domain", "item"],

bin/lib/objects/Images.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import base64
+import os
+import sys
+
+from hashlib import sha256
+from io import BytesIO
+
+from flask import url_for
+from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
+
+config_loader = ConfigLoader()
+r_serv_metadata = config_loader.get_db_conn("Kvrocks_Objects")
+IMAGE_FOLDER = config_loader.get_files_directory('images')
+config_loader = None
+
+
+class Image(AbstractDaterangeObject):
+    """
+    AIL Screenshot Object. (strings)
+    """
+
+    # ID = SHA256
+    def __init__(self, image_id):
+        super(Image, self).__init__('image', image_id)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def exists(self):
+        return os.path.isfile(self.get_filepath())
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        return {'style': 'far', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius': 5}
+
+    def get_rel_path(self):
+        rel_path = os.path.join(self.id[0:2], self.id[2:4], self.id[4:6], self.id[6:8], self.id[8:10], self.id[10:12], self.id[12:])
+        return rel_path
+
+    def get_filepath(self):
+        filename = os.path.join(IMAGE_FOLDER, self.get_rel_path())
+        return os.path.realpath(filename)
+
+    def get_file_content(self):
+        filepath = self.get_filepath()
+        with open(filepath, 'rb') as f:
+            file_content = BytesIO(f.read())
+        return file_content
+
+    def get_content(self, r_type='str'):
+        return self.get_file_content()
+
+    def get_misp_object(self):
+        obj_attrs = []
+        obj = MISPObject('file')
+
+        obj_attrs.append(obj.add_attribute('sha256', value=self.id))
+        obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content()))
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['id'] = self.id
+        meta['img'] = self.id
+        meta['tags'] = self.get_tags(r_list=True)
+        if 'content' in options:
+            meta['content'] = self.get_content()
+        if 'tags_safe' in options:
+            meta['tags_safe'] = self.is_tags_safe(meta['tags'])
+        return meta
+
+    def create(self, content):
+        filepath = self.get_filepath()
+        dirname = os.path.dirname(filepath)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filepath, 'wb') as f:
+            f.write(content)
+
+def get_screenshot_dir():
+    return IMAGE_FOLDER
+
+
+def create(content, size_limit=5000000, b64=False, force=False):
+    size = (len(content)*3) / 4
+    if size <= size_limit or size_limit < 0 or force:
+        if b64:
+            content = base64.standard_b64decode(content.encode())
+        image_id = sha256(content).hexdigest()
+        image = Image(image_id)
+        if not image.exists():
+            image.create(content)
+        return image
+
+
+class Images(AbstractDaterangeObjects):
+    """
+        CookieName Objects
+    """
+    def __init__(self):
+        super().__init__('image', Image)
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search  # TODO
+
+
+# if __name__ == '__main__':
+#     name_to_search = '29ba'
+#     print(search_screenshots_by_name(name_to_search))

bin/lib/objects/Messages.py

@@ -88,7 +88,7 @@ class Message(AbstractObject):
 
     def get_timestamp(self):
         dirs = self.id.split('/')
-        return dirs[-2]
+        return dirs[1]
 
     def get_message_id(self):  # TODO optimize
         message_id = self.get_basename().rsplit('/', 1)[1]
@@ -104,6 +104,14 @@ class Message(AbstractObject):
     # TODO get channel ID
     # TODO get thread  ID
 
+    def get_images(self):
+        images = []
+        for child in self.get_childrens():
+            obj_type, _, obj_id = child.split(':', 2)
+            if obj_type == 'image':
+                images.append(obj_id)
+        return images
+
     def get_user_account(self, meta=False):
         user_account = self.get_correlation('user-account')
         if user_account.get('user-account'):
@@ -194,7 +202,7 @@ class Message(AbstractObject):
         else:
             timestamp = float(timestamp)
         timestamp = datetime.fromtimestamp(float(timestamp))
-        meta['date'] = timestamp.strftime('%Y%/m/%d')
+        meta['date'] = timestamp.strftime('%Y/%m/%d')
         meta['hour'] = timestamp.strftime('%H:%M:%S')
         meta['full_date'] = timestamp.isoformat(' ')
 
@@ -222,6 +230,8 @@ class Message(AbstractObject):
                 meta['user-account'] = {'id': 'UNKNOWN'}
         if 'chat' in options:
             meta['chat'] = self.get_chat_id()
+        if 'images' in options:
+            meta['images'] = self.get_images()
 
         # meta['encoding'] = None
         return meta

bin/lib/objects/abstract_chat_object.py

@@ -128,8 +128,18 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
     def get_nb_messages(self):
         return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
 
-    def _get_messages(self):  # TODO paginate
+    def _get_messages(self, nb=-1, page=1):
-        return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
+        if nb < 1:
+            return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
+        else:
+            if page > 1:
+                start = page - 1 + nb
+            else:
+                start = 0
+            messages = r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True)
+            if messages:
+                messages = reversed(messages)
+            return messages
 
     def get_timestamp_first_message(self):
         return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0, withscores=True)
@@ -169,15 +179,15 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
 
     def get_message_meta(self, message, timestamp=None):  # TODO handle file message
         message = Messages.Message(message[9:])
-        meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
+        meta = message.get_meta(options={'content', 'images', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
         return meta
 
-    def get_messages(self, start=0, page=1, nb=500, unread=False):  # threads ????
+    def get_messages(self, start=0, page=1, nb=500, unread=False):  # threads ???? # TODO ADD last/first message timestamp + return page
         # TODO return message meta
         tags = {}
         messages = {}
         curr_date = None
-        for message in self._get_messages():
+        for message in self._get_messages(nb=10, page=3):
             timestamp = message[1]
             date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
             if date_day != curr_date:

bin/lib/objects/abstract_daterange_object.py

@@ -71,8 +71,8 @@ class AbstractDaterangeObject(AbstractObject, ABC):
         else:
             return last_seen
 
-    def get_nb_seen(self):
+    def get_nb_seen(self): # TODO REPLACE ME -> correlation image
-        return self.get_nb_correlation('item')
+        return self.get_nb_correlation('item') + self.get_nb_correlation('message')
 
     def get_nb_seen_by_date(self, date):
         nb = r_object.zscore(f'{self.type}:date:{date}', self.id)
@@ -125,7 +125,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
     def _add_create(self):
         r_object.sadd(f'{self.type}:all', self.id)
 
-    def _add(self, date, obj):
+    def _add(self, date, obj): # TODO OBJ=None
         if not self.exists():
             self._add_create()
             self.set_first_seen(date)
@@ -134,13 +134,12 @@ class AbstractDaterangeObject(AbstractObject, ABC):
             self.update_daterange(date)
         update_obj_date(date, self.type)
 
+        r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
+
         if obj:
             # Correlations
             self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id())
 
-            # Stats NB by day: # TODO Don't increase on reprocess
-            r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
-
             if obj.type == 'item':
                 item_id = obj.get_id()
                 # domain

bin/lib/objects/ail_objects.py

@@ -23,6 +23,7 @@ from lib.objects import Etags
 from lib.objects.Favicons import Favicon
 from lib.objects import HHHashs
 from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
+from lib.objects import Images
 from lib.objects.Messages import Message
 from lib.objects import Pgps
 from lib.objects.Screenshots import Screenshot
@@ -70,6 +71,8 @@ def get_object(obj_type, subtype, obj_id):
         return Favicon(obj_id)
     elif obj_type == 'hhhash':
         return HHHashs.HHHash(obj_id)
+    elif obj_type == 'image':
+        return Images.Image(obj_id)
     elif obj_type == 'message':
         return Message(obj_id)
     elif obj_type == 'screenshot':

configs/core.cfg.sample

@@ -6,6 +6,7 @@ hash = HASHS
 crawled = crawled
 har = CRAWLED_SCREENSHOT
 screenshot = CRAWLED_SCREENSHOT/screenshot
+images = IMAGES
 
 wordtrending_csv = var/www/static/csv/wordstrendingdata
 wordsfile = files/wordfile

var/www/Flask_server.py

@@ -51,6 +51,7 @@ from blueprints.objects_cookie_name import objects_cookie_name
 from blueprints.objects_etag import objects_etag
 from blueprints.objects_hhhash import objects_hhhash
 from blueprints.chats_explorer import chats_explorer
+from blueprints.objects_image import objects_image
 
 Flask_dir = os.environ['AIL_FLASK']
 
@@ -109,6 +110,7 @@ app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
 app.register_blueprint(objects_etag, url_prefix=baseUrl)
 app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
 app.register_blueprint(chats_explorer, url_prefix=baseUrl)
+app.register_blueprint(objects_image, url_prefix=baseUrl)
 
 # =========       =========#
 

var/www/blueprints/objects_image.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects import Images
+from packages import Date
+
+# ============ BLUEPRINT ============
+objects_image = Blueprint('objects_image', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/image'))
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+
+# ============ FUNCTIONS ============
+@objects_image.route('/image/<path:filename>')
+@login_required
+@login_read_only
+@no_cache
+def image(filename):
+    if not filename:
+        abort(404)
+    if not 64 <= len(filename) <= 70:
+        abort(404)
+    filename = filename.replace('/', '')
+    image = Images.Image(filename)
+    return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True)
+
+
+@objects_image.route("/objects/images", methods=['GET'])
+@login_required
+@login_read_only
+def objects_images():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    show_objects = request.args.get('show_objects')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+
+    if show_objects:
+        dict_objects = Images.Images().api_get_meta_by_daterange(date_from, date_to)
+    else:
+        dict_objects = {}
+
+    print(dict_objects)
+
+    return render_template("ImageDaterange.html", date_from=date_from, date_to=date_to,
+                           dict_objects=dict_objects, show_objects=show_objects)
+
+
+@objects_image.route("/objects/images/post", methods=['POST'])
+@login_required
+@login_read_only
+def objects_images_post():
+    date_from = request.form.get('date_from')
+    date_to = request.form.get('date_to')
+    show_objects = request.form.get('show_objects')
+    return redirect(url_for('objects_image.objects_images', date_from=date_from, date_to=date_to, show_objects=show_objects))
+
+
+@objects_image.route("/objects/images/range/json", methods=['GET'])
+@login_required
+@login_read_only
+def objects_images_range_json():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+    return jsonify(Images.Images().api_get_chart_nb_by_daterange(date_from, date_to))
+
+# ============= ROUTES ==============
+

var/www/templates/chats_explorer/chat_viewer.html

@@ -35,6 +35,10 @@
             height: 2px;
             background: #eee;
         }
+        .message_image {
+            max-width: 50%;
+            filter: blur(5px);
+        }
 	</style>
 
 </head>
@@ -147,6 +151,30 @@
 
                 {% if chat['messages'] %}
 
+                    <span class="mt-3">
+                        <div class="card border-secondary">
+                            <div class="card-body py-2">
+                                <div class="row">
+                                    <div class="col-md-3 text-center px-0">
+                                        <button class="btn btn-sm btn-secondary" onclick="blur_slider.val(0);blur_images();">
+                                            <i class="fas fa-eye-slash"></i>
+                                            <span class="label-icon">Hide</span>
+                                        </button>
+                                    </div>
+                                    <div class="col-md-6 text-center pl-0 pt-1">
+                                        <input type="range" min="0" max="15" step="0.1" value="10" id="blur-slider" onchange="blur_images();">
+                                    </div>
+                                    <div class="col-md-3 text-center">
+                                        <button class="btn btn-sm btn-secondary" onclick="blur_slider.val(15);blur_images();">
+                                            <i class="fas fa-image"></i>
+                                            <span class="label-icon">Full</span>
+                                        </button>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                    </span>
+
                     <div class="position-relative">
                         <div class="chat-messages p-2">
 
@@ -193,6 +221,9 @@
     {#                                                </div>#}
                                                 </div>
                                             {% endif %}
+                                            {% if mess['images'] %}
+                                                <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=mess['images'][0])}}">
+                                            {% endif %}
                                             <pre class="my-0">{{ mess['content'] }}</pre>
                                             {% for tag in mess['tags'] %}
                                                 <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
@@ -245,6 +276,17 @@ function toggle_sidebar(){
 		$('#core_content').addClass('col-lg-10')
 	}
 }
+
+
+const blur_slider = $('#blur-slider');
+function blur_images(){
+    let blurValue = blur_slider.val();
+    blurValue = 15 - blurValue;
+    let images = document.getElementsByClassName('message_image');
+    for(i = 0; i < images.length; i++) {
+        images[i].style.filter = "blur(" + blurValue + "px)";
+    }
+}
 </script>
 
 
@@ -405,9 +447,6 @@ d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid
         tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
     }
     const mouseleave = function(d) {
-        console.log(d)
-        console.log(d.hour)
-        console.log(d.day)
         tooltip.style("opacity", 0)
         d3.select(this)
             .style("stroke", "none")

var/www/templates/correlation/show_correlation.html

@@ -684,7 +684,12 @@ if (d.popover) {
 
 					if (data["img"]) {
                         if (data["tags_safe"]) {
-						    desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}" + data["img"] +" class=\"img-thumbnail blured\" id=\"tooltip_screenshot_correlation\" style=\"\"/>";
+                            if (data["type"] === "screenshot") {
+						        desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
+                            } else {
+                                desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
+                            }
+                            desc = desc + data["img"] +" class=\"img-thumbnail blured\" id=\"tooltip_screenshot_correlation\" style=\"\"/>";
                         } else {
                             desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
                         }

var/www/templates/objects/image/ImageDaterange.html

@@ -0,0 +1,602 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Images - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+	<style>
+		.input-group .form-control {
+			position: unset;
+		}
+		.line {
+  		fill: none;
+  		stroke: #000;
+  		stroke-width: 2.0px;
+		}
+		.bar {
+      fill: steelblue;
+    }
+    .bar:hover{
+      fill: brown;
+			cursor: pointer;
+    }
+		.bar_stack:hover{
+			cursor: pointer;
+    }
+		.pie_path:hover{
+			cursor: pointer;
+		}
+		.svgText {
+  		pointer-events: none;
+		}
+		div.tooltip {
+			position: absolute;
+			text-align: center;
+			padding: 2px;
+			font: 12px sans-serif;
+			background: #ebf4fb;
+			border: 2px solid #b7ddf2;
+			border-radius: 8px;
+			pointer-events: none;
+			color: #000000;
+		}
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-xl-10">
+						<div class="mt-1" id="barchart_type"></div>
+
+{#                        {% include 'image/block_images_search.html' %}#}
+
+                    </div>
+
+
+					<div class="col-xl-2">
+
+						<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
+						  <div class="card-body text-center py-2">
+						    <h6 class="card-title" style="color:#286090;">Select a date range :</h6>
+								<form action="{{ url_for('objects_image.objects_images_post') }}" id="hash_selector_form" method='post'>
+									<div class="input-group" id="date-range-from">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
+									</div>
+									<div class="input-group" id="date-range-to">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
+									</div>
+									<div class="form-check my-1">
+										<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
+										<label class="form-check-label" for="checkbox-input-show">
+											<span style="color:#286090; font-size: 14px;">
+												Show Images <i class="fas fa-key"></i>
+											</span>
+										</label>
+									</div>
+									<button class="btn btn-primary" style="text-align:center;">
+										<i class="fas fa-copy"></i> Search
+									</button>
+								</form>
+						  </div>
+						</div>
+
+						<div id="pie_chart_encoded">
+						</div>
+						<div id="pie_chart_top5_types">
+						</div>
+					</div>
+				</div>
+
+				{% if dict_objects %}
+                    {% if date_from|string == date_to|string %}
+                        <h3> {{ date_from }} Images Name: </h3>
+                    {% else %}
+                        <h3> {{ date_from }} to {{ date_to }} Images Name: </h3>
+                    {% endif %}
+                    <table id="tableb64" class="table table-striped table-bordered">
+                        <thead class="bg-dark text-white">
+                        <tr>
+                            <th></th>
+                            <th>First Seen</th>
+                            <th>Last Seen</th>
+                            <th>Total</th>
+                            <th>Last days</th>
+                        </tr>
+                        </thead>
+						<tbody style="font-size: 15px;">
+                        {% for obj_id in dict_objects %}
+                            <tr>
+                                <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=image&id={{ obj_id }}">{{ dict_objects[obj_id]['id'] }}</a></td>
+                                <td>{{ dict_objects[obj_id]['first_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['last_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
+                                <td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+
+
+                {% else %}
+                    {% if show_objects %}
+                        {% if date_from|string == date_to|string %}
+							<h3> {{ date_from }}, No Image</h3>
+                        {% else %}
+							<h3> {{ date_from }} to {{ date_to }}, No Image</h3>
+                        {% endif %}
+                    {% endif %}
+                {% endif %}
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_image").addClass("active");
+
+        $('#date-range-from').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+        $('#date-range-to').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+
+        $('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+        $('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+
+        $('#tableb64').DataTable({
+            "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        });
+
+
+        chart.stackBarChart = barchart_type_stack("{{ url_for('objects_image.objects_images_range_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
+
+
+        chart.onResize();
+        $(window).on("resize", function() {
+            chart.onResize();
+        });
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+<script>
+	{% for obj_id in dict_objects %}
+    sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
+	{% endfor %}
+</script>
+
+<script>
+var margin = {top: 20, right: 100, bottom: 55, left: 45},
+				width  = 1000 - margin.left - margin.right,
+				height = 500  - margin.top  - margin.bottom;
+var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+
+var y = d3.scaleLinear().rangeRound([height, 0]);
+
+var xAxis = d3.axisBottom(x);
+
+var yAxis = d3.axisLeft(y);
+
+var color = d3.scaleOrdinal(d3.schemeSet3);
+
+var svg = d3.select("#barchart_type").append("svg")
+				.attr("id", "thesvg")
+				.attr("viewBox", "0 0 1000 500")
+				.attr("width",  width  + margin.left + margin.right)
+				.attr("height", height + margin.top  + margin.bottom)
+			.append("g")
+				.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+function barchart_type_stack(url, id) {
+
+  d3.json(url)
+    .then(function(data){
+
+			var labelVar = 'date';  //A
+		  var varNames = d3.keys(data[0])
+		      .filter(function (key) { return key !== labelVar;}); //B
+
+		  data.forEach(function (d) { //D
+		    var y0 = 0;
+		    d.mapping = varNames.map(function (name) {
+		      return {
+		        name: name,
+		        label: d[labelVar],
+		        y0: y0,
+		        y1: y0 += +d[name]
+		      };
+		    });
+		    d.total = d.mapping[d.mapping.length - 1].y1;
+		  });
+
+		  x.domain(data.map(function (d) { return (d.date); })); //E
+		  y.domain([0, d3.max(data, function (d) { return d.total; })]);
+
+			svg.append("g")
+        .attr("class", "x axis")
+        .attr("transform", "translate(0," + height + ")")
+        .call(xAxis)
+				.selectAll("text")
+					.attr("class", "bar")
+					{% if  date_from|string == date_to|string and type is none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_image.objects_images') }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
+						.attr("transform", "rotate(-18)" )
+					{% elif  date_from|string == date_to|string and type is not none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_image.objects_images') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-18)" )
+					{% else %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_image.objects_images') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-40)" )
+					{% endif %}
+					.style("text-anchor", "end");
+
+    	svg.append("g")
+        .attr("class", "y axis")
+        .call(yAxis)
+      	.append("text")
+        	.attr("transform", "rotate(-90)")
+        	.attr("y", 6)
+        	.attr("dy", ".71em")
+        	.style("text-anchor", "end");
+
+			var selection = svg.selectAll(".series")
+		    .data(data)
+		    .enter().append("g")
+		      .attr("class", "series")
+		      .attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
+
+			selection.selectAll("rect")
+	      .data(function (d) { return d.mapping; })
+	    .enter().append("rect")
+				.attr("class", "bar_stack")
+	      .attr("width", x.bandwidth())
+	      .attr("y", function (d) { return y(d.y1); })
+	      .attr("height", function (d) { return y(d.y0) - y(d.y1); })
+	      .style("fill", function (d) { return color(d.name); })
+	      .style("stroke", "grey")
+	      .on("mouseover", function (d) { showPopover.call(this, d); })
+				.on("mouseout", function (d) { removePopovers(); })
+				{% if  date_from|string == date_to|string and type is none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_image.objects_images') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
+				{% elif  date_from|string == date_to|string and type is not none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_image.objects_images') }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
+				{% else %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_image.objects_images') }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
+				{% endif %}
+
+				data.forEach(function(d) {
+					if(d.total !== 0){
+						svg.append("text")
+								.attr("class", "bar")
+								.attr("dy", "-.35em")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+								.attr('y', y(d.total))
+								{% if  date_from|string == date_to|string and type is none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_image.objects_images') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
+								{% elif  date_from|string == date_to|string and type is not none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_image.objects_images') }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
+								{% else %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_image.objects_images') }}"+'?date_from='+d.date+'&date_to='+d.date })
+								{% endif %}
+								.style("text-anchor", "middle")
+								.text(d.total);
+					}
+				});
+
+			drawLegend(varNames);
+    });
+
+}
+
+function drawLegend (varNames) {
+    var legend = svg.selectAll(".legend")
+        .data(varNames.slice().reverse())
+      .enter().append("g")
+        .attr("class", "legend")
+        .attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
+
+    legend.append("rect")
+        .attr("x", 943)
+        .attr("width", 10)
+        .attr("height", 10)
+        .style("fill", color)
+        .style("stroke", "grey");
+
+    legend.append("text")
+				.attr("class", "svgText")
+        .attr("x", 941)
+        .attr("y", 6)
+        .attr("dy", ".35em")
+        .style("text-anchor", "end")
+        .text(function (d) { return d; });
+}
+
+function removePopovers () {
+    $('.popover').each(function() {
+      $(this).remove();
+    });
+  }
+
+function showPopover (d) {
+  $(this).popover({
+    title: "<b><span id='tooltip-id-name-bar'></span></b>",
+    placement: 'top',
+    container: 'body',
+    trigger: 'manual',
+    html : true,
+    content: function() {
+      return "<span id='tooltip-id-label'></span>" +
+            "<br/>num: <span id='tooltip-id-value-bar'></span>"; }
+  });
+  $(this).popover('show');
+  $("#tooltip-id-name-bar").text(d.name);
+  $("#tooltip-id-label").text(d.label);
+  $("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
+}
+
+chart.onResize = function () {
+    var aspect = 1000 / 500, chart = $("#thesvg");
+    var targetWidth = chart.parent().width();
+    chart.attr("width", targetWidth);
+    chart.attr("height", targetWidth / aspect);
+  }
+
+window.chart = chart;
+
+</script>
+
+<script>
+function draw_pie_chart(id, url_json, pie_on_click_url) {
+
+		var width_pie = 200;
+		var height_pie = 200;
+		var padding_pie = 10;
+		var opacity_pie = .8;
+
+		var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
+		//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
+		var color_pie = d3.scaleOrdinal(d3.schemeSet3);
+
+		var div_pie = d3.select("body").append("div")
+				.attr("class", "tooltip")
+				.style("opacity", 0);
+
+		var svg_pie = d3.select("#"+id)
+				.append('svg')
+				.attr("width", '100%')
+				.attr("height", '100%')
+				.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
+				.attr('preserveAspectRatio','xMinYMin')
+
+
+		var g_pie = svg_pie.append('g')
+				.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
+
+		var arc_pie = d3.arc()
+				.innerRadius(0)
+				.outerRadius(radius_pie);
+
+		d3.json(url_json)
+			.then(function(data){
+
+			var pie_pie = d3.pie()
+					.value(function(d) { return d.value; })
+					.sort(null);
+
+			var path_pie = g_pie.selectAll('path')
+				  .data(pie_pie(data))
+				  .enter()
+				  .append("g")
+				  .append('path')
+				  .attr('d', arc_pie)
+				  .attr('fill', (d,i) => color_pie(i))
+					.attr('class', 'pie_path')
+					.on("mouseover", mouseovered_pie)
+					.on("mouseout", mouseouted_pie)
+					.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
+				  .style('opacity', opacity_pie)
+				  .style('stroke', 'white');
+			});
+
+
+		function mouseovered_pie(d) {
+      //remove old content
+      $("#tooltip-id-name").remove();
+      $("#tooltip-id-value").remove();
+
+			// tooltip
+			var content;
+
+			content = "<b><span id='tooltip-id-name'></span></b><br/>"+
+								"<br/>"+
+								"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
+
+			div_pie.transition()
+				.duration(200)
+				.style("opacity", .9);
+			div_pie.html(content)
+				.style("left", (d3.event.pageX) + "px")
+				.style("top", (d3.event.pageY - 28) + "px");
+
+      $("#tooltip-id-name").text(d.data.name);
+      $("#tooltip-id-value").text(d.data.value);
+		}
+
+		function mouseouted_pie() {
+			div_pie.transition()
+				.duration(500)
+				.style("opacity", 0);
+		}
+}
+
+</script>
+
+
+
+<script>
+function barchart_type(url, id) {
+
+
+  var margin = {top: 20, right: 20, bottom: 70, left: 40};
+
+  var width = 960 - margin.left - margin.right;
+  var height = 500 - margin.top - margin.bottom;
+
+  var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+  var y = d3.scaleLinear().rangeRound([height, 0]);
+
+  var xAxis = d3.axisBottom(x)
+    //.tickFormat(d3.time.format("%Y-%m"));
+
+  var yAxis = d3.axisLeft(y)
+    .ticks(10);
+
+/*var svg = d3.select(id).append("svg")
+      .attr("width", width + margin.left + margin.right)
+      .attr("height", height + margin.top + margin.bottom)
+			.attr("id", "thesvg")
+      .append("g")
+        .attr("transform",
+          "translate(" + margin.left + "," + margin.top + ")");*/
+
+
+  d3.json(url)
+    .then(function(data){
+
+        data.forEach(function(d) {
+          d.value = +d.value;
+        });
+
+				x.domain(data.map(function(d) { return d.date; }));
+        y.domain([0, d3.max(data, function(d) { return d.value; })]);
+
+        var label = svg.append("g")
+            .attr("class", "x axis")
+            .attr("transform", "translate(0," + height + ")")
+            .call(xAxis)
+          .selectAll("text")
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", "-.55em")
+						{% if  daily_type_chart %}
+						.attr("transform", "rotate(-20)" );
+						{% else %}
+						.attr("transform", "rotate(-70)" )
+						.attr("class", "bar")
+						{% endif %}
+
+        svg.append("g")
+            .attr("class", "y axis")
+            .call(yAxis)
+          .append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", 6)
+            .attr("dy", ".71em")
+            .style("text-anchor", "end")
+            .text("Value ($)");
+
+        var bar = svg.selectAll("bar")
+            .data(data)
+          .enter().append("rect")
+            .attr("class", "bar")
+            //.style("fill", "steelblue")
+						.attr("x", function(d) { return x(d.date); })
+            .attr("width", x.bandwidth())
+            .attr("y", function(d) { return y(d.value); })
+            .attr("height", function(d) { return height - y(d.value); })
+
+
+        data.forEach(function(d) {
+          if(d.value != 0){
+            svg.append("text")
+                .attr("class", "bar")
+                .attr("dy", "-.35em")
+                //.text(function(d) { return d.value; });
+                .text(d.value)
+								.style("text-anchor", "middle")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+                .attr('y', y(d.value));
+          }
+        });
+
+    });
+
+}
+</script>
+
+
+</body>
+
+</html>