Merge branch 'relationships' into otp

bin/importer/feeders/Default.py

@@ -44,6 +44,11 @@ class DefaultFeeder:
     def get_date(self):
         return datetime.date.today().strftime("%Y%m%d")
 
+    def get_feeder_timestamp(self):
+        timestamp = self.json_data.get('timestamp')
+        if timestamp:
+            return int(timestamp)
+
     def get_json_data(self):
         """
         Return the JSON data,

bin/importer/feeders/abstract_chats_feeder.py

@@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
 import datetime
 import os
 import sys
+import time
 
 from abc import ABC
 
@@ -127,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         return self.json_data['meta'].get('reply_to', {}).get('message_id')
 
     def get_message_forward(self):
-        return self.json_data['meta'].get('forward')
+        return self.json_data['meta'].get('forward', {})
 
     def get_message_content(self):
         decoded = base64.standard_b64decode(self.json_data['data'])
@@ -164,7 +165,40 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             self.obj = Messages.Message(obj_id)
         return self.obj
 
-    def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
+    # TODO handle subchannel
+    def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
+        chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
+
+        # Obj Daterange
+        chat.add(date)
+
+        if meta_chat.get('name'):
+            chat.set_name(meta_chat['name'])
+
+        if meta_chat.get('info'):
+            chat.set_info(meta_chat['info'])
+
+        if meta_chat.get('date'): # TODO check if already exists
+            chat.set_created_at(int(meta_chat['date']['timestamp']))
+
+        if meta_chat.get('icon'):
+            img = Images.create(meta_chat['icon'], b64=True)
+            img.add(date, chat)
+            chat.set_icon(img.get_global_id())
+            if new_objs:
+                new_objs.add(img)
+
+        if meta_chat.get('username'):
+            username = Username(meta_chat['username'], self.get_chat_protocol())
+            chat.update_username_timeline(username.get_global_id(), int(time.time()))
+            username.add(date, obj=chat)  # TODO TODAY DATE
+
+        return chat
+
+    ##############################################################################################################################
+
+
+    def process_chat(self, new_objs, obj, date, timestamp, feeder_timestamp, reply_id=None):
         meta = self.json_data['meta']['chat'] # todo replace me by function
         chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
         subchannel = None
@@ -190,7 +224,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
 
         if meta.get('username'):
             username = Username(meta['username'], self.get_chat_protocol())
-            chat.update_username_timeline(username.get_global_id(), timestamp)
+            chat.update_username_timeline(username.get_global_id(), feeder_timestamp)
 
         if meta.get('subchannel'):
             subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
@@ -257,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         # else:
         #   # ADD NEW MESSAGE REF (used by discord)
 
+    ##########################################################################################
+
+    def _process_user(self, meta, date, timestamp, new_objs=None):
+        user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
+        if meta.get('username'):
+            username = Username(meta['username'], self.get_chat_protocol())
+            # TODO timeline or/and correlation ????
+            user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+            user_account.update_username_timeline(username.get_global_id(), timestamp)
+
+            # Username---Message
+            username.add(date)  # TODO # correlation message ??? ###############################################################
+
+        # ADDITIONAL METAS
+        if meta.get('firstname'):
+            user_account.set_first_name(meta['firstname'])
+        if meta.get('lastname'):
+            user_account.set_last_name(meta['lastname'])
+        if meta.get('phone'):
+            user_account.set_phone(meta['phone'])
+
+        if meta.get('icon'):
+            img = Images.create(meta['icon'], b64=True)
+            img.add(date, user_account)
+            user_account.set_icon(img.get_global_id())
+            new_objs.add(img)
+
+        if meta.get('info'):
+            user_account.set_info(meta['info'])
+
+        user_account.add(date)
+        return user_account
+
     def process_sender(self, new_objs, obj, date, timestamp):
         meta = self.json_data['meta'].get('sender')
         if not meta:
@@ -305,8 +372,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         if self.obj:
             objs.add(self.obj)
         new_objs = set()
+        chats_objs = set()
 
         date, timestamp = self.get_message_date_timestamp()
+        feeder_timestamp = self.get_feeder_timestamp()
+        if not feeder_timestamp:
+            feeder_timestamp = timestamp
 
         # REPLY
         reply_id = self.get_message_reply_id()
@@ -363,81 +434,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             print(obj.id)
 
             # CHAT
-            chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
+            curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, feeder_timestamp, reply_id=reply_id)
-
-            # # TODO HANDLE OTHERS OBJECT TYPE
-            # # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-            # # Message forward + Discussion
-            # if self.get_json_meta().get('forward'):
-            #     discussion_id = self.get_json_meta().get('discussion')
-            #     forward_from = self.get_message_forward()
-            #
-            #     if discussion_id:       # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
-            #         chat_forward_id = forward_from['from']['id']
-            #         message_forward_id = forward_from['from']['channel_post']
-            #
-            #         # if chat_forward_id == discussion_id:
-            #         #     linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
-            #         #     if linked_chat.exists():
-            #         #         # create thread
-            #         #         # add message replies for each childrens
-            #
-            # # TODO HANDLE THREAD
-            # # TODO Change FORWARD META FIELDS
-            # # meta['forward'] = {}
-            # #       # CHAT ID
-            # #       # SUBCHANNEL ID     -> can be None
-            # #       # Message ID
-            #
-            # # meta['forward']['origin']
-            # #       # same as 'forward'
-            #
-            # if self.get_json_meta().get('forward'):
-            #     forward = self.get_message_forward()
-            #     f_chat = forward['chat']
-            #     f_subchannel = forward.get('subchannel')
-            #     f_id = forward.get('id')
-            #     if not f_subchannel:
-            #         chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
-            #         if chat_forward.exists():
-            #             for chat_obj in chat_objs:
-            #                 if chat_obj.type == 'chat':
-            #                     chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
-            #             # TODO LIST FORWARDED MESSAGES
-            #
-            #
-            # # Discord -> serverID + subchannel ID + message ID
-            # # Telegram -> chat ID + Message ID
-            # #                 + ORIGIN IDs
-            #
-            #
-            #
-            # # TODO create relationships graph
-            #
-            #
-            # # TODO REMOVE ME
-            # # Message forward  # TODO handle subchannel + message ID
-            # if self.get_json_meta().get('forward'):
-            #     forward_from = self.get_message_forward()
-            #     print('-----------------------------------------------------------')
-            #     print(forward_from)
-            #     if forward_from:
-            #         forward_from_type = forward_from['from']['type']
-            #         if forward_from_type == 'channel' or forward_from_type == 'chat':
-            #             chat_forward_id = forward_from['from']['id']
-            #             chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
-            #             if chat_forward.exists():
-            #                 for chat_obj in chat_objs:
-            #                     if chat_obj.type == 'chat':
-            #                         chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
-            #                         # chat_forward.add_relationship(obj.get_global_id(), 'forward')
 
             # SENDER # TODO HANDLE NULL SENDER
-            user_account = self.process_sender(new_objs, obj, date, timestamp)
+            user_account = self.process_sender(new_objs, obj, date, feeder_timestamp)
 
             if user_account:
             # UserAccount---ChatObjects
-                for obj_chat in chat_objs:
+                for obj_chat in curr_chats_objs:
                     user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
 
             # if chat: # TODO Chat---Username correlation ???
@@ -449,4 +453,73 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
                 #       -> subchannel ?
                 #       -> thread id ?
 
+            chats_objs.update(curr_chats_objs)
+
+        #######################################################################
+
+        ## FORWARD ##
+        chat_fwd = None
+        user_fwd = None
+        if self.get_json_meta().get('forward'):
+            meta_fwd = self.get_message_forward()
+            if meta_fwd.get('chat'):
+                chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
+                for chat_obj in chats_objs:
+                    if chat_obj.type == 'chat':
+                        chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+            if meta_fwd.get('user'):
+                user_fwd = self._process_user(meta_fwd['user'], date, feeder_timestamp, new_objs=new_objs)  # TODO date, timestamp ???
+                for chat_obj in chats_objs:
+                    if chat_obj.type == 'chat':
+                        user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+
+        # TODO chat_fwd -> message
+        if chat_fwd or user_fwd:
+            for obj in objs:
+                if obj.type == 'message':
+                    if chat_fwd:
+                        obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
+                    if user_fwd:
+                        obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
+                    for chat_obj in chats_objs:
+                        if chat_obj.type == 'chat':
+                            obj.add_relationship(chat_obj.get_global_id(), 'in')
+        # -FORWARD- #
+
+        ## MENTION ##
+        if self.get_json_meta().get('mentions'):
+            for mention in self.get_json_meta()['mentions'].get('chats', []):
+                m_obj = self._process_chat(mention, date, new_objs=new_objs)
+                if m_obj:
+                    for chat_obj in chats_objs:
+                        if chat_obj.type == 'chat':
+                            chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+                    # TODO PERF
+                    # TODO Keep message obj + chat obj in global var
+                    for obj in objs:
+                        if obj.type == 'message':
+                            obj.add_relationship(m_obj.get_global_id(), 'mention')
+                            for chat_obj in chats_objs:
+                                if chat_obj.type == 'chat':
+                                    obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+            for mention in self.get_json_meta()['mentions'].get('users', []):
+                m_obj = self._process_user(mention, date, feeder_timestamp, new_objs=new_objs)  # TODO date ???
+                if m_obj:
+                    for chat_obj in chats_objs:
+                        if chat_obj.type == 'chat':
+                            chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+                    # TODO PERF
+                    # TODO Keep message obj + chat obj in global var
+                    for obj in objs:
+                        if obj.type == 'message':
+                            obj.add_relationship(m_obj.get_global_id(), 'mention')
+                            for chat_obj in chats_objs:
+                                if chat_obj.type == 'chat':
+                                    obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+         # -MENTION- #
+
         return new_objs | objs

bin/lib/Tracker.py

@@ -762,6 +762,9 @@ def delete_obj_trackers(obj_type, subtype, obj_id):
 #### TRACKERS ACL ####
 
 ## LEVEL ##
+def is_tracker_global_level(tracker_uuid):
+    return int(r_tracker.hget(f'tracker:{tracker_uuid}', 'level')) == 1
+
 def is_tracked_in_global_level(tracked, tracker_type):
     for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
         tracker = Tracker(tracker_uuid)
@@ -805,6 +808,19 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
         return {"status": "error", "reason": "Access Denied"}, 403
     return {"uuid": tracker_uuid}, 200
 
+
+def api_is_allowed_to_access_tracker(tracker_uuid, user_id):
+    if not is_valid_uuid_v4(tracker_uuid):
+        return {"status": "error", "reason": "Invalid uuid"}, 400
+    tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
+    if not tracker_creator:
+        return {"status": "error", "reason": "Unknown uuid"}, 404
+    user = User(user_id)
+    if not is_tracker_global_level(tracker_uuid):
+        if not user.is_in_role('admin') and user_id != tracker_creator:
+            return {"status": "error", "reason": "Access Denied"}, 403
+    return {"uuid": tracker_uuid}, 200
+
 ##-- ACL --##
 
 #### FIX DB #### TODO ###################################################################

bin/lib/ail_core.py

@@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type):  # TODO Dynamic subtype
     if obj_type == 'chat-thread':
         return r_object.smembers(f'all_chat-thread:subtypes')
     if obj_type == 'cryptocurrency':
-        return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+        return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
     if obj_type == 'pgp':
         return ['key', 'mail', 'name']
     if obj_type == 'username':

bin/lib/chats_viewer.py

@@ -328,6 +328,31 @@ def get_username_meta_from_global_id(username_global_id):
     username = Usernames.Username(username_id, instance_uuid)
     return username.get_meta(options={'icon'})
 
+###############################################################################
+# TODO Pagination
+def list_messages_to_dict(l_messages_id, translation_target=None):
+    options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
+    meta = {}
+    curr_date = None
+    for mess_id in l_messages_id:
+        message = Messages.Message(mess_id[1:])
+        timestamp = message.get_timestamp()
+        date_day = message.get_date()
+        date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}'
+        if date_day != curr_date:
+            meta[date_day] = []
+            curr_date = date_day
+        meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
+        meta[date_day].append(meta_mess)
+
+        # if mess_dict.get('tags'):
+        #     for tag in mess_dict['tags']:
+        #         if tag not in tags:
+        #             tags[tag] = 0
+        #         tags[tag] += 1
+    # return messages, pagination, tags
+    return meta
+
 # TODO Filter
 ## Instance type
 ## Chats IDS
@@ -354,7 +379,7 @@ def get_messages_iterator(filters={}):
             # threads
             for threads in chat.get_threads():
                 thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
-                _, _ = thread._get_messages(nb=-1)
+                messages, _ = thread._get_messages(nb=-1)
                 for mess in messages:
                     message_id, _, message_id = mess[0].split(':', )
                     yield Messages.Message(message_id)
@@ -404,6 +429,15 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
         meta.append(chat_meta)
     return meta
 
+def get_user_account_chat_message(user_id, subtype, chat_id):  # TODO subchannel + threads ...
+    meta = {}
+    chat = Chats.Chat(chat_id, subtype)
+    chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
+    if chat_meta['username']:
+        chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
+
+    meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None)
+    return meta
 
 def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
     week = {}
@@ -431,6 +465,60 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
         nb_day += 1
     return stats
 
+def get_user_account_chats_chord(subtype, user_id):
+    nb = {}
+    user_account = UsersAccount.UserAccount(user_id, subtype)
+    for chat_g_id in user_account.get_chats():
+        c_subtype, c_id = chat_g_id.split(':', 1)
+        chat = Chats.Chat(c_id, c_subtype)
+        nb[f'chat:{chat_g_id}'] = len(chat.get_user_messages(user_id))
+
+    user_account_gid = user_account.get_global_id() # # #
+    chord = {'meta': {}, 'data': []}
+    label = get_chat_user_account_label(user_account_gid)
+    if label:
+        chord['meta'][user_account_gid] = label
+    else:
+        chord['meta'][user_account_gid] = user_account_gid
+
+    for chat_g_id in nb:
+        label = get_chat_user_account_label(chat_g_id)
+        if label:
+            chord['meta'][chat_g_id] = label
+        else:
+            chord['meta'][chat_g_id] = chat_g_id
+        chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
+    return chord
+
+def get_user_account_mentions_chord(subtype, user_id):
+    chord = {'meta': {}, 'data': []}
+    nb = {}
+    user_account = UsersAccount.UserAccount(user_id, subtype)
+    user_account_gid = user_account.get_global_id()
+    label = get_chat_user_account_label(user_account_gid)
+    if label:
+        chord['meta'][user_account_gid] = label
+    else:
+        chord['meta'][user_account_gid] = user_account_gid
+
+    for mess in user_account.get_messages():
+        m = Messages.Message(mess[9:])
+        for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
+            if rel:
+                if not rel['target'] in nb:
+                    nb[rel['target']] = 0
+                nb[rel['target']] += 1
+
+    for g_id in nb:
+        label = get_chat_user_account_label(g_id)
+        if label:
+            chord['meta'][g_id] = label
+        else:
+            chord['meta'][g_id] = g_id
+        chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
+    return chord
+
+
 def _get_chat_card_meta_options():
     return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
 
@@ -481,18 +569,65 @@ def fix_correlations_subchannel_message():
 
 #### API ####
 
+def get_chat_user_account_label(chat_gid):
+    label = None
+    obj_type, subtype, obj_id = chat_gid.split(':', 2)
+    if obj_type == 'chat':
+        obj = get_obj_chat(obj_type, subtype, obj_id)
+        username = obj.get_username()
+        if username:
+            username = username.split(':', 2)[2]
+        name = obj.get_name()
+        if username and name:
+            label = f'{username} - {name}'
+        elif username:
+            label = username
+        elif name:
+            label = name
+
+    elif obj_type == 'user-account':
+        obj = UsersAccount.UserAccount(obj_id, subtype)
+        username = obj.get_username()
+        if username:
+            username = username.split(':', 2)[2]
+        name = obj.get_name()
+        if username and name:
+            label = f'{username} - {name}'
+        elif username:
+            label = username
+        elif name:
+            label = name
+    return label
+
+def enrich_chat_relationships_labels(relationships):
+    meta = {}
+    for row in relationships:
+        if row['source'] not in meta:
+            label = get_chat_user_account_label(row['source'])
+            if label:
+                meta[row['source']] = label
+            else:
+                meta[row['source']] = row['source']
+
+        if row['target'] not in meta:
+            label = get_chat_user_account_label(row['target'])
+            if label:
+                meta[row['target']] = label
+            else:
+                meta[row['target']] = row['target']
+    return meta
 def api_get_chat_service_instance(chat_instance_uuid):
     chat_instance = ChatServiceInstance(chat_instance_uuid)
     if not chat_instance.exists():
         return {"status": "error", "reason": "Unknown uuid"}, 404
     return chat_instance.get_meta({'chats'}), 200
 
-def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
+def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True):
     chat = Chats.Chat(chat_id, chat_instance_uuid)
     if not chat.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
     # print(chat.get_obj_language_stats())
-    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
+    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target)
     if meta['username']:
         meta['username'] = get_username_meta_from_global_id(meta['username'])
     if meta['subchannels']:
@@ -500,7 +635,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
     else:
         if translation_target not in Language.get_translation_languages():
             translation_target = None
-        meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
+        if messages:
+            meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
     return meta, 200
 
 def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id):
@@ -602,6 +738,18 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
         meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
     return meta, 200
 
+def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None):
+    user_account = UsersAccount.UserAccount(user_id, instance_uuid)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    meta = get_user_account_chat_message(user_id, instance_uuid, chat_id)
+    meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
+    resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False)
+    if resp[1] != 200:
+        return resp
+    meta['chat'] = resp[0]
+    return meta, 200
+
 def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
     user_account = UsersAccount.UserAccount(user_id, instance_uuid)
     if not user_account.exists():

bin/lib/crawlers.py

@@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he
     schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
     return schedule_uuid
 
+def _delete_schedules():
+    for schedule_uuid in get_schedulers_uuid():
+        schedule = CrawlerSchedule(schedule_uuid)
+        schedule.delete()
+
 # TODO sanityze UUID
 def api_delete_schedule(data):
     schedule_uuid = data.get('uuid')
@@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
                             external=external)
     return task_uuid
 
-
 ## -- CRAWLER TASK -- ##
 
 #### CRAWLER TASK API ####

bin/lib/objects/CryptoCurrencies.py

@@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject):
         pass
 
     def is_valid_address(self):
-        if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin':
+        if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron':
             return check_base58_address(self.id)
         else:
             return True
@@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject):
             return 'ZEC'
         elif self.subtype == 'dash':
             return 'DASH'
+        elif self.subtype == 'tron':
+            return 'TRX'
         return None
 
     def get_link(self, flask_context=False):
@@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject):
 
 def get_all_subtypes():
     # return ail_core.get_object_all_subtypes(self.type)
-    return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+    return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
 
 
 # def build_crypto_regex(subtype, search_id):
@@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol):
         return 'zcash'
     elif symbol == 'DASH':
         return 'dash'
+    elif symbol == 'TRX':
+        return 'tron'
     return None
 
 
@@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype):
 def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
     if subtype == '':
         pass
-    elif subtype == 'name':
-        pass
-    elif subtype == 'mail':
-        pass
     return name_to_search
 
 def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):

bin/lib/objects/UsersAccount.py

@@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
     def get_last_name(self):
         return self._get_field('lastname')
 
+    def get_name(self):
+        first_name = self.get_first_name()
+        last_name = self.get_last_name()
+        if first_name and last_name:
+            return f'{first_name} {last_name}'
+        elif first_name:
+            return first_name
+        elif last_name:
+            return last_name
+
     def get_phone(self):
         return self._get_field('phone')
 
@@ -129,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
 
     def get_messages(self):
         messages = []
-        for mess in self.get_correlation('message'):
+        correl = self.get_correlation('message')
-            messages.append(f'message:{mess}')
+        if 'message' in correl:
+            for mess_id in correl['message']:
+                messages.append(f'message:{mess_id}')
         return messages
 
-
     def get_messages_by_chat_obj(self, chat_obj):
         messages = []
         for mess in self.get_correlation_iter_obj(chat_obj, 'message'):
@@ -157,6 +168,7 @@ class UserAccount(AbstractSubtypeObject):
             meta['usernames'] = self.get_usernames()
         if 'icon' in options:
             meta['icon'] = self.get_icon()
+            meta['svg_icon'] = meta['icon']
         if 'info' in options:
             meta['info'] = self.get_info()
             if 'translation' in options and translation_target:

bin/lib/objects/abstract_object.py

@@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
-from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
+from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
 from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
@@ -299,6 +299,9 @@ class AbstractObject(ABC):
     def get_nb_relationships(self, filter=[]):
         return get_obj_nb_relationships(self.get_global_id())
 
+    def get_obj_relationships(self, relationships=set(), filter_types=set()):
+        return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
+
     def add_relationship(self, obj2_global_id, relationship, source=True):
         # is source
         if source:

bin/lib/objects/ail_objects.py

@@ -562,24 +562,47 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
 
 # --- CORRELATION --- #
 
+#### RELATIONSHIPS ####
+
+def get_relationships():
+    return relationships_engine.get_relationships()
+
+def sanitize_relationships(relationships):
+    return relationships_engine.sanitize_relationships(relationships)
 def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.get_nb_relationships(filter=filter_types)
 
-def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
                                  objs_hidden=set(),
                                  flask_context=False):
     obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
-    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
+    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
-                                                                    filter_types=filter_types,
+                                                                     filter_types=filter_types,
-                                                                    max_nodes=max_nodes, level=level,
+                                                                     max_nodes=max_nodes, level=level,
-                                                                    objs_hidden=objs_hidden)
+                                                                     objs_hidden=objs_hidden)
     # print(meta)
     meta['objs'] = list(meta['objs'])
     return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
             "links": links,
             "meta": meta}
 
+def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
+    if obj_type == 'chat':
+        obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+        data = relationships_engine.get_chat_forward_stats(obj_global_id)
+        return data
+    return []
+
+def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
+    if obj_type == 'chat':
+        obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+        data = relationships_engine.get_chat_mentions_stats(obj_global_id)
+        return data
+    return []
+
+# --- RELATIONSHIPS --- #
+
 
 # if __name__ == '__main__':
     # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

bin/lib/relationships_engine.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
-
+import json
 import os
 import sys
 
@@ -16,41 +16,103 @@ config_loader = None
 
 
 RELATIONSHIPS = {
-    "forward",
+    "forwarded_from",
+    "forwarded_to",  # forwarded_to
+    "in",
     "mention"
 }
+
+RELATIONSHIPS_OBJS = { # TODO forward user-account
+    "forwarded_from": {
+        'chat': {'message'},
+        'message': {'chat', 'user-account'}
+    },
+    "forwarded_to": {
+        'chat': {'chat'},
+        'user-account': {'chat'},
+    },
+    "in": {
+        'chat': {'message'},
+        'message': {'chat'}
+    },
+    "mention": {
+        'chat': {'chat', 'user-account', 'message'},
+        'message': {'chat', 'user-account'},
+        'user-account': {'chat', 'message'},
+    },
+}
+
 def get_relationships():
     return RELATIONSHIPS
 
-
+def sanitize_relationships(relationships):
-def get_obj_relationships_by_type(obj_global_id, relationship):
+    rels = get_relationships()
-    return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
+    if relationships:
-
+        relationships = set(relationships).intersection(rels)
-def get_obj_nb_relationships_by_type(obj_global_id, relationship):
+    if not relationships:
-    return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
+        relationships = rels
-
+        if not relationships:
-def get_obj_relationships(obj_global_id):
+            return []
-    relationships = []
-    for relationship in get_relationships():
-        for rel in get_obj_relationships_by_type(obj_global_id, relationship):
-            meta = {'relationship': relationship}
-            direction, obj_id = rel.split(':', 1)
-            if direction == 'i':
-                meta['source'] = obj_id
-                meta['target'] = obj_global_id
-            else:
-                meta['target'] = obj_id
-                meta['source'] = obj_global_id
-
-            if not obj_id.startswith('chat'):
-                continue
-
-            meta['id'] = obj_id
-            # meta['direction'] = direction
-            relationships.append(meta)
     return relationships
 
-def get_obj_nb_relationships(obj_global_id):
+def get_relationship_obj_types(relationship):
+    return RELATIONSHIPS_OBJS.get(relationship, {})
+
+def get_relationship_objs(relationship, obj_type):
+    return get_relationship_obj_types(relationship).get(obj_type, set())
+
+def sanityze_obj_types(relationship, obj_type, filter_types):
+    objs_types = get_relationship_objs(relationship, obj_type)
+    if filter_types:
+        filter_types = objs_types.intersection(filter_types)
+    else:
+        filter_types = objs_types
+    # if not filter_types:
+    #     filter_types = objs_types
+    #     if not filter_types:
+    #         return []
+    return filter_types
+
+# TODO check obj_type
+# TODO sanitize relationships
+
+def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+    obj_type = obj_global_id.split(':', 1)[0]
+    relationships = {}
+    filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+    for o_type in filter_types:
+        relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
+    return relationships
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+    obj_type = obj_global_id.split(':', 1)[0]
+    relationships = {}
+    filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+    for o_type in filter_types:
+        relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
+    return relationships
+
+def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
+    all_relationships = []
+    for relationship in relationships:
+        obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
+        for obj_type in obj_relationships:
+            for rel in obj_relationships[obj_type]:
+                meta = {'relationship': relationship}
+                direction, obj_id = rel.split(':', 1)
+                if direction == 'i':
+                    meta['source'] = obj_id
+                    meta['target'] = obj_global_id
+                else:
+                    meta['target'] = obj_id
+                    meta['source'] = obj_global_id
+
+                meta['id'] = obj_id
+                # meta['direction'] = direction
+                all_relationships.append(meta)
+    return all_relationships
+
+def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
     nb = {}
     for relationship in get_relationships():
         nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
@@ -59,27 +121,27 @@ def get_obj_nb_relationships(obj_global_id):
 
 # TODO Filter by obj type ???
 def add_obj_relationship(source, target, relationship):
-    r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
+    source_type = source.split(':', 1)[0]
-    r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
+    target_type = target.split(':', 1)[0]
-    # r_rel.sadd(f'rels:{source}', relationship)
+    r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
-    # r_rel.sadd(f'rels:{target}', relationship)
+    r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
 
 
-def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
     links = []
     nodes = set()
     meta = {'complete': True, 'objs': set()}
     done = set()
     done_link = set()
 
-    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
     return nodes, links, meta
 
-def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
     meta['objs'].add(obj_global_id)
     nodes.add(obj_global_id)
 
-    for rel in get_obj_relationships(obj_global_id):
+    for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
         meta['objs'].add(rel['id'])
 
         if rel['id'] in done:
@@ -99,13 +161,119 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
         if level > 0:
             next_level = level - 1
 
-            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
 
     # done.add(rel['id'])
 
+# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
 
-if __name__ == '__main__':
+def get_chat_forward_stats_out(obj_global_id):
-    source = ''
+    nb = {}
-    target = ''
+    for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
-    add_obj_relationship(source, target, 'forward')
+        chat_mess = rel['source'].split('/')
-    # print(get_obj_relationships(source))
+
+        chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+        if chat_target not in nb:
+            nb[chat_target] = 0
+        nb[chat_target] += 1
+    return nb
+
+def get_chat_forward_stats_in(obj_global_id):
+    nb = {}
+    for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+        r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
+        if r:
+            if not r[0]['target'] in nb:
+                nb[r[0]['target']] = 0
+            nb[r[0]['target']] += 1
+    #     chat_mess = rel['source'].split('/')
+    #
+    #     chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+    #     print(chat_target, chat, chat_target == chat)
+    #     if chat is None or chat_target == chat:
+    #         if chat_target not in nb:
+    #             nb[chat_target] = 0
+    #         nb[chat_target] += 1
+    #
+    # print(json.dumps(nb, indent=4))
+    return nb
+
+def get_chat_forward_stats(obj_global_id): # objs_hidden
+    data = []
+
+    # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+    in_mess = get_chat_forward_stats_out(obj_global_id)
+
+    out_mess = get_chat_forward_stats_in(obj_global_id)
+
+
+        # if rel['target'] == obj_global_id:
+        #     # print(rel)
+        #     nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
+    #
+    for target in in_mess:
+        data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+    for source in out_mess:
+        data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+    # print()
+    # print(in_mess)
+    # print()
+    # print(out_mess)
+
+    return data
+
+##################################################################
+
+def get_chat_mention_stats_in(obj_global_id):
+    nb = {}
+    for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
+        chat_mess = rel['source'].split('/')
+        chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+        if chat_source not in nb:
+            nb[chat_source] = 0
+        nb[chat_source] += 1
+    return nb
+
+def get_chat_mention_stats_out(obj_global_id):
+    nb = {}
+    for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+        r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
+        if r:
+            if not r[0]['target'] in nb:
+                nb[r[0]['target']] = 0
+            nb[r[0]['target']] += 1
+    #     chat_mess = rel['source'].split('/')
+    #
+    #     chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+    #     print(chat_target, chat, chat_target == chat)
+    #     if chat is None or chat_target == chat:
+    #         if chat_target not in nb:
+    #             nb[chat_target] = 0
+    #         nb[chat_target] += 1
+    #
+    # print(json.dumps(nb, indent=4))
+    return nb
+
+def get_chat_mentions_stats(obj_global_id): # objs_hidden
+    data = []
+
+    # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+    out_mess = get_chat_mention_stats_in(obj_global_id)
+    in_mess = get_chat_mention_stats_out(obj_global_id)
+    #
+    for target in in_mess:
+        data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+    for source in out_mess:
+        data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+    # print()
+    # print(in_mess)
+    # print()
+    # print(out_mess)
+
+    return data

bin/modules/Cryptocurrencies.py

@@ -92,7 +92,13 @@ CURRENCIES = {
         'regex': r'\b(?<![+/=])X[A-Za-z0-9]{33}(?![+/=])\b',
         'max_execution_time': default_max_execution_time,
         'tag': 'infoleak:automatic-detection="dash-address"',
-    }
+    },
+    'tron': {
+            'name': 'tron',  # e.g. TYdds9VLDjUshf9tbsXSfGUZNzJSbbBeat
+            'regex': r'\b(?<![+/=])T[0-9a-zA-Z]{33}(?![+/=])\b',
+            'max_execution_time': default_max_execution_time,
+            'tag': 'infoleak:automatic-detection="tron-address"',
+        },
 }
 ##################################
 ##################################

bin/packages/Date.py

@@ -263,3 +263,9 @@ def sanitise_daterange(date_from, date_to, separator='', date_type='str'):
         date_from = date_to
         date_to = res
     return date_from, date_to
+
+def get_previous_month_date():
+    now = datetime.date.today()
+    first = now.replace(day=1)
+    last_month = first - datetime.timedelta(days=1)
+    return last_month.strftime("%Y%m%d")

var/www/blueprints/chats_explorer.py

@@ -297,6 +297,44 @@ def objects_user_account():
                                ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
                                translation_languages=languages, translation_target=target)
 
+@chats_explorer.route("/objects/user-account_chats_chord_json", methods=['GET']) # TODO API
+@login_required
+@login_read_only
+def objects_user_account_chats_chord_json():
+    subtype = request.args.get('subtype')
+    user_id = request.args.get('id')
+    json_graph = chats_viewer.get_user_account_chats_chord(subtype, user_id)
+    return jsonify(json_graph)
+
+@chats_explorer.route("/objects/user-account_mentions_chord_json", methods=['GET']) # TODO API
+@login_required
+@login_read_only
+def objects_user_account_mentions_chord_json():
+    subtype = request.args.get('subtype')
+    user_id = request.args.get('id')
+    json_graph = chats_viewer.get_user_account_mentions_chord(subtype, user_id)
+    return jsonify(json_graph)
+
+@chats_explorer.route("/objects/user-account/chat", methods=['GET'])
+@login_required
+@login_read_only
+def objects_user_account_chat():
+    instance_uuid = request.args.get('subtype')
+    user_id = request.args.get('id')
+    chat_id = request.args.get('chat_id')
+    target = request.args.get('target')
+    if target == "Don't Translate":
+        target = None
+    meta = chats_viewer.api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=target)
+    if meta[1] != 200:
+        return create_json_response(meta[0], meta[1])
+    else:
+        meta = meta[0]
+        languages = Language.get_translation_languages()
+        return render_template('chats_explorer/user_chat_messages.html', meta=meta, bootstrap_label=bootstrap_label,
+                               ail_tags=Tag.get_modal_add_tags(meta['user-account']['id'], meta['user-account']['type'], meta['user-account']['subtype']),
+                               translation_languages=languages, translation_target=target)
+
 @chats_explorer.route("objects/user-account/messages/stats/week/all", methods=['GET'])
 @login_required
 @login_read_only

var/www/blueprints/correlation.py

@@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.objects import ail_objects
+from lib import chats_viewer
 from lib import Tag
 
 bootstrap_label = Flask_config.bootstrap_label
@@ -263,9 +264,37 @@ def relationships_graph_node_json():
     max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
     level = sanitise_level(request.args.get('level'))
 
-    json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
+    filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
+    relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
+
+    json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
     return jsonify(json_graph)
 
+@correlation.route('/relationships/chord_graph_json')
+@login_required
+@login_read_only
+def relationships_chord_graph_json():
+    obj_id = request.args.get('id')
+    subtype = request.args.get('subtype')
+    obj_type = request.args.get('type')
+
+    chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
+    meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
+
+    return jsonify({'meta': meta, 'data': chat_json_graph})
+
+@correlation.route('/relationships/chord_mentions_graph_json')
+@login_required
+@login_read_only
+def relationships_chord_mentions_graph_json():
+    obj_id = request.args.get('id')
+    subtype = request.args.get('subtype')
+    obj_type = request.args.get('type')
+
+    chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
+    meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
+
+    return jsonify({'meta': meta, 'data': chat_json_graph})
 
 @correlation.route('/relationship/show', methods=['GET', 'POST'])
 @login_required
@@ -278,8 +307,28 @@ def show_relationship():
         max_nodes = request.form.get('max_nb_nodes_in')
         level = sanitise_level(request.form.get('level'))
 
+        ## get all selected relationships
+        relationships = []
+        for relationship in ail_objects.get_relationships():
+            rel_option = request.form.get(f'relationship_{relationship}_Check')
+            if rel_option:
+                relationships.append(relationship)
+
+        relationships = ",".join(relationships)
+
+        ## get all selected objects types
+        filter_types = []
+        for ob_type in ail_objects.get_all_objects():
+            correl_option = request.form.get(f'{ob_type}_Check')
+            if correl_option:
+                filter_types.append(ob_type)
+
+        # list as params
+        filter_types = ",".join(filter_types)
+
         # redirect to keep history and bookmark
         return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
+                                filter=filter_types, relationships=relationships,
                                 max_nodes=max_nodes, level=level))
 
     # request.method == 'GET'
@@ -290,6 +339,9 @@ def show_relationship():
         max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
         level = sanitise_level(request.args.get('level'))
 
+        filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
+        relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
+
         # check if obj_id exist
         if not ail_objects.exists_obj(obj_type, subtype, obj_id):
             return abort(404)
@@ -300,6 +352,8 @@ def show_relationship():
                            "object_type": obj_type,
                            "max_nodes": max_nodes, "level": level,
                            "correlation_id": obj_id,
+                           "relationships": relationships, "relationships_str": ",".join(relationships),
+                           "filter": filter_types, "filter_str": ",".join(filter_types),
                            "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
                            "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
                            }

var/www/blueprints/crawler_splash.py

@@ -316,6 +316,17 @@ def crawlers_last_domains_month_json():
     stats = crawlers.get_crawlers_stats_by_month(domain_type)
     return jsonify(stats)
 
+@crawler_splash.route('/crawlers/last/domains/month/previous/json')
+@login_required
+@login_read_only
+def crawlers_last_domains_previous_month_json():
+    domain_type = request.args.get('type')
+    if domain_type not in crawlers.get_crawler_all_types():
+        return jsonify({'error': 'Invalid domain type'}), 400
+    date = Date.get_previous_month_date()
+    stats = crawlers.get_crawlers_stats_by_month(domain_type, date=date)
+    return jsonify(stats)
+
 @crawler_splash.route('/crawlers/last/domains/status/month/json')
 @login_required
 @login_read_only
@@ -800,7 +811,7 @@ def crawler_cookiejar_cookie_edit_post():
 @login_required
 @login_read_only
 def crawler_cookiejar_cookie_add():
-    cookiejar_uuid = request.args.get('cookiejar_uuid')
+    cookiejar_uuid = request.args.get('uuid')
     return render_template("add_cookie.html", cookiejar_uuid=cookiejar_uuid)
 
 
@@ -831,7 +842,7 @@ def crawler_cookiejar_cookie_manual_add_post():
     if res[1] != 200:
         return create_json_response(res[0], res[1])
 
-    return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
+    return redirect(url_for('crawler_splash.crawler_cookiejar_show', uuid=cookiejar_uuid))
 
 
 @crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['POST'])

var/www/blueprints/hunters.py

@@ -145,7 +145,7 @@ def tracked_menu_admin():
 def show_tracker():
     user_id = current_user.get_user_id()
     tracker_uuid = request.args.get('uuid', None)
-    res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
+    res = Tracker.api_is_allowed_to_access_tracker(tracker_uuid, user_id)
     if res[1] != 200:  # invalid access
         return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
@@ -159,7 +159,7 @@ def show_tracker():
 
     tracker = Tracker.Tracker(tracker_uuid)
     meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
-                                     'user', 'webhook', 'nb_objs'})
+                                     'user', 'webhooks', 'nb_objs'})
 
     if meta['type'] == 'yara':
         yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
@@ -300,6 +300,7 @@ def add_tracked_menu():
             return create_json_response(res[0], res[1])
     else:
         return render_template("tracker_add.html",
+                               dict_tracker={},
                                all_sources=item_basic.get_all_items_sources(r_list=True),
                                tags_selector_data=Tag.get_tags_selector_data(),
                                all_yara_files=Tracker.get_all_default_yara_files())
@@ -314,6 +315,8 @@ def tracker_edit():
         res = Tracker.api_edit_tracker(input_dict, user_id)
         if res[1] == 200:
             return redirect(url_for('hunters.show_tracker', uuid=res[0].get('uuid')))
+        else:
+            return create_json_response(res[0], res[1])
     else:
         user_id = current_user.get_user_id()
         tracker_uuid = request.args.get('uuid', None)
@@ -322,10 +325,16 @@ def tracker_edit():
             return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
         tracker = Tracker.Tracker(tracker_uuid)
-        dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhook'})
+        dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhooks'})
         if dict_tracker['type'] == 'yara':
             if not Tracker.is_default_yara_rule(dict_tracker['tracked']):
                 dict_tracker['content'] = Tracker.get_yara_rule_content(dict_tracker['tracked'])
+        elif dict_tracker['type'] == 'set':
+            tracked, nb_words = dict_tracker['tracked'].rsplit(';', 1)
+            tracked = tracked.replace(',', ' ')
+            dict_tracker['tracked'] = tracked
+            dict_tracker['nb_words'] = nb_words
+
         taxonomies_tags, galaxies_tags, custom_tags = Tag.sort_tags_taxonomies_galaxies_customs(dict_tracker['tags'])
         tags_selector_data = Tag.get_tags_selector_data()
         tags_selector_data['taxonomies_tags'] = taxonomies_tags

var/www/static/js/d3/chord_directed_diagram.js

@@ -0,0 +1,140 @@
+//Requirement:	- D3v7
+//				- jquery
+
+// container_id = #container_id{"meta": {obj_gid: label, ...}, "data": [{"source": source, "target": target, "value": value}, ...]}
+// tooltip = d3 tooltip object
+
+// TODO: - Mouseover object
+
+const create_directed_chord_diagram = (container_id, data, min_value= 0, max_value = -1, fct_mouseover, fct_mouseout, options) => {
+
+    // Filter data by value between target and source
+    if (min_value > 0) {
+        data.data = data.data.filter(function(value) {
+            return data.value >= min_value;
+        });
+    }
+    if (max_value > 0) {
+        data.data = data.data.filter(function(value) {
+            return data.value <= max_value;
+        });
+    }
+
+
+    function getMaxCharsToShow(angle, radius) {
+        const approximateCharWidth = 7; // Approximate width of a character in pixels
+        const arcLength = angle * radius; // Length of the arc
+        return Math.floor(arcLength / approximateCharWidth); // Maximum number of characters that can fit
+    }
+
+    const width = 900;
+    const height = width;
+    const innerRadius = Math.min(width, height) * 0.5 - 20;
+    const outerRadius = innerRadius + 6;
+
+    const labels_meta = data.meta
+    data = data.data
+
+    // Compute a dense matrix from the weighted links in data.
+    var names = Array.from(d3.union(data.flatMap(d => [d.source, d.target])));
+    const index = new Map(names.map((name, i) => [name, i]));
+    const matrix = Array.from(index, () => new Array(names.length).fill(0));
+    for (const {source, target, value} of data) matrix[index.get(source)][index.get(target)] += value;
+
+    const chord = d3.chordDirected()
+        .padAngle(12 / innerRadius)
+        .sortSubgroups(d3.descending)
+        .sortChords(d3.descending);
+
+    const arc = d3.arc()
+        .innerRadius(innerRadius)
+        .outerRadius(outerRadius);
+
+    const ribbon = d3.ribbonArrow()
+        .radius(innerRadius - 0.5)
+        .padAngle(1 / innerRadius);
+
+    const colors = d3.schemeCategory10;
+
+    const svg = d3.select(container_id)
+        .append("svg")
+            .attr("width", width)
+            .attr("height", height)
+            .attr("viewBox", [-width / 2, -height / 2, width, height])
+            .attr("style", "width: 100%; height: auto; font: 10px sans-serif;");
+
+    const chords = chord(matrix);
+
+    const textId = `text-${Math.random().toString(36).substring(2, 15)}`;
+
+    svg.append("path")
+        .attr("id", textId)
+        .attr("fill", "none")
+        .attr("d", d3.arc()({outerRadius, startAngle: 0, endAngle: 2 * Math.PI}));
+
+    svg.append("g")
+        .attr("fill-opacity", 0.75)
+        .selectAll("path")
+        .data(chords)
+        .enter()
+        .append("path")
+            .attr("d", ribbon)
+            .attr("fill", d => colors[d.target.index])
+            .style("mix-blend-mode", "multiply")
+        .append("title")
+            .text(d => `${labels_meta[names[d.source.index]]}
+=> ${d.source.value} Messages
+${labels_meta[names[d.target.index]]}`);
+
+    const g = svg.append("g")
+        .selectAll("g")
+        .data(chords.groups)
+        .enter()
+        .append("g");
+
+    g.append("path")
+        .attr("d", arc)
+        .attr("fill", d => colors[d.index])
+        .attr("stroke", "#fff")
+        .on("mouseover", function(event, d) {
+            fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]]);
+        })
+	    .on("mouseout", fct_mouseout);
+
+    g.each(function(d) {
+        const group = d3.select(this);
+        const angle = d.endAngle - d.startAngle;
+        const text = labels_meta[names[d.index]];
+        const maxCharsToShow = getMaxCharsToShow(angle, outerRadius);
+
+        let displayedText
+        if (maxCharsToShow <= 1) {
+            displayedText = text[0];
+        } else {
+            displayedText = text.length > maxCharsToShow ? text.slice(0, maxCharsToShow - 1) + "…" : text;
+        }
+
+        let info_text = "OUT: " + d3.sum(matrix[d.index]) + " ; IN: " + d3.sum(matrix, row => row[d.index]) + " Messages"
+
+        if (displayedText) {
+            group.append("text")
+                .attr("dy", -3)
+                .append("textPath")
+                .attr("xlink:href", `#${textId}`)
+                .attr("startOffset", d.startAngle * outerRadius)
+                .on("mouseover", function(event, d) {
+                    fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]], info_text);
+                })
+                .on("mouseout", fct_mouseout)
+                .text(displayedText);
+        }
+    });
+
+    //return svg.node();
+}
+
+
+
+// return svg ???
+	// return svg
+

var/www/static/js/d3/heatmap_week_hour.js

@@ -1,4 +1,4 @@
-//Requirement:	- D3v5
+//Requirement:	- D3v7
 //				- jquery
 // data input: var data = [{"count":0,"date":"2023-11-20","day":0,"hour":0}
 // based on gist nbremer/62cf60e116ae821c06602793d265eaf6
@@ -33,10 +33,12 @@ const create_heatmap_week_hour = (container_id, data, options) => {
         .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
 
     // create a tooltip
-    const tooltip = d3.select(container_id)
+    const tooltip = d3.select("body")
         .append("div")
+        .attr("class", "tooltip_heatmap")
         .style("opacity", 0)
-        .attr("class", "tooltip")
+        .style("position", "absolute")
+        .style("pointer-events", "none")
         .style("background-color", "white")
         .style("border", "solid")
         .style("border-width", "2px")
@@ -44,24 +46,28 @@ const create_heatmap_week_hour = (container_id, data, options) => {
         .style("padding", "5px")
 
     // Three function that change the tooltip when user hover / move / leave a cell
-    const mouseover = function(d) {
+    const mouseover = function(event, d) {
-        tooltip.style("opacity", 1)
+        d3.select(event.target)
-        d3.select(this)
             .style("stroke", "black")
-            //.style("stroke-opacity", 1)
+            .style("stroke-opacity", 1)
 
-        var xPosition = d3.mouse(this)[0] + margin.left;
+        let d3_pageX = event.pageX;
-        var yPosition = d3.mouse(this)[1] + margin.top + window.scrollY + 100;
+        let d3_pageY = event.pageY;
 
         tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
-            .style("left", xPosition + "px")
+            .style("left", (d3_pageX) + "px")
-            .style("top", yPosition + "px");
+		    .style("top", (d3_pageY - 28) + "px");
+        tooltip.transition()
+		    .duration(200)
+		    .style("opacity", 1);
     }
-    const mouseleave = function(d) {
+    const mouseleave = function(event, d) {
-        tooltip.style("opacity", 0)
+        d3.select(event.target)
-        d3.select(this)
             .style("stroke", "white")
             //.style("stroke-opacity", 0.8)
+        tooltip.transition()
+		    .duration(200)
+            .style("opacity", 0)
     }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -114,22 +120,22 @@ const create_heatmap_week_hour = (container_id, data, options) => {
     var heatMap = svg.selectAll(".hour")
         .data(data)
         .enter().append("rect")
-        .attr("x", function (d) {
+            .attr("x", function (d) {
-            return (d.hour - 1) * gridSize;
+                return (d.hour - 1) * gridSize;
-        })
+            })
-        .attr("y", function (d) {
+            .attr("y", function (d) {
-            return (d.day - 1) * gridSize;
+                return (d.day - 1) * gridSize;
-        })
+            })
-        .attr("class", "hour bordered")
+            .attr("class", "hour bordered")
-        .attr("width", gridSize)
+            .attr("width", gridSize)
-        .attr("height", gridSize)
+            .attr("height", gridSize)
-        .style("stroke", "white")
+            .style("stroke", "white")
-        .style("stroke-opacity", 0.6)
+            .style("stroke-opacity", 0.6)
-        .style("fill", function (d) {
+            .style("fill", function (d) {
-            return colorScale(d.count);
+                return colorScale(d.count);
-        })
+            })
-        .on("mouseover", mouseover)
+            .on("mouseover", mouseover)
-        .on("mouseleave", mouseleave);
+            .on("mouseleave", mouseleave);
 
 //Append title to the top
     svg.append("text")
@@ -222,7 +228,6 @@ const create_heatmap_week_hour = (container_id, data, options) => {
         .attr("transform", "translate(0," + (10) + ")")
         .call(xAxis);
     
-
 // return svg ???
 	// return svg
 

var/www/static/js/helper.js

@@ -1,4 +1,4 @@
 // sanitise str_to_sanitize
 function sanitize_text(str_to_sanitize) {
     return $("<span\>").text(str_to_sanitize).html()
-};
+}

var/www/templates/chats_explorer/basic_card_chat.html

@@ -85,7 +85,7 @@
                                 <i class="fas fa-user-circle"></i>
                                 <i class="far fa-comment-dots"></i>
                             </span>
-                            {{ meta["nb_messages"] }}&nbsp;&nbsp;
+                            <a class="badge-primary px-1 py-0" href="{{ url_for('chats_explorer.objects_user_account_chat') }}?subtype={{ meta['subtype'] }}&id={{ main_obj_id }}&chat_id={{ meta['id'] }}">{{ meta["nb_messages"] }}&nbsp;&nbsp;</a>
                         </span>
                     {% endif %}
                 </div>

var/www/templates/chats_explorer/card_chat.html

@@ -95,6 +95,10 @@
 {#                        <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
                     </button>
                 </a>
+                <a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
+                    <button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships &nbsp;
+                    </button>
+                </a>
             {% endif %}
         </span>
 

var/www/templates/chats_explorer/card_user_account.html

@@ -87,6 +87,11 @@
             {% else %}
                 <a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
                     <button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
+{#                        <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
+                    </button>
+                </a>
+                <a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
+                    <button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships &nbsp;
 {#                        <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
                     </button>
                 </a>

var/www/templates/chats_explorer/user_account.html

@@ -12,12 +12,14 @@
 
   <!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/helper.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
-    <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
 
 
 </head>
@@ -34,6 +36,9 @@
 
                 {% include 'chats_explorer/card_user_account.html' %}
 
+                <span class="mt-3">
+                    {% include 'objects/image/block_blur_img_slider.html' %}
+                </span>
                 {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
                     {% include 'chats_explorer/block_translation.html' %}
                 {% endwith %}
@@ -42,10 +47,16 @@
                     <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>
                     <div id="heatmapweekhourall"></div>
 
+                    <h4>Numbers of Messages Posted by Chat:</h4>
+                    <div id="chord_user_chats" style="max-width: 900px"></div>
+
+                    <h4>Numbers of Mentions:</h4>
+                    <div id="chord_mentions" style="max-width: 900px"></div>
+
                     <h4>User Chats:</h4>
                     {% for meta_chats in meta['chats'] %}
                         <div class="my-2">
-                            {% with meta=meta_chats  %}
+                            {% with meta=meta_chats,main_obj_id=meta['id']  %}
                                 {% include 'chats_explorer/basic_card_chat.html' %}
                             {% endwith %}
                         </div>
@@ -58,6 +69,8 @@
         </div>
     </div>
 
+
+{% include 'objects/tooltip_ail_objects.html' %}
 <script>
     $(document).ready(function(){
         $("#page-Decoded").addClass("active");
@@ -77,26 +90,19 @@ d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?su
         create_heatmap_week_hour('#heatmapweekhourall', data);
     })
 
-function toggle_sidebar(){
+let url = "{{ url_for('chats_explorer.objects_user_account_chats_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
-	if($('#nav_menu').is(':visible')){
+d3.json(url).then(function(data) {
-		$('#nav_menu').hide();
+    create_directed_chord_diagram('#chord_user_chats', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
-		$('#side_menu').removeClass('border-right')
+});
-		$('#side_menu').removeClass('col-lg-2')
+
-		$('#core_content').removeClass('col-lg-10')
+let url2 = "{{ url_for('chats_explorer.objects_user_account_mentions_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
-	}else{
+d3.json(url2).then(function(data) {
-		$('#nav_menu').show();
+    create_directed_chord_diagram('#chord_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
-		$('#side_menu').addClass('border-right')
+});
-		$('#side_menu').addClass('col-lg-2')
-		$('#core_content').addClass('col-lg-10')
-	}
-}
 
 </script>
 
 
-
-
-
 </body>
 
 </html>

var/www/templates/chats_explorer/user_chat_messages.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>User Account - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
+
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <h3>User:</h3>
+                {% with meta=meta['user-account'] %}
+                    {% include 'chats_explorer/card_user_account.html' %}
+                {% endwith %}
+
+                <h3>Chat:</h3>
+                {% with meta=meta['chat']  %}
+                    {% include 'chats_explorer/basic_card_chat.html' %}
+                {% endwith %}
+
+                <div class="mt-2">
+                {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
+                    {% include 'chats_explorer/block_translation.html' %}
+                {% endwith %}
+                {% include 'objects/image/block_blur_img_slider.html' %}
+                </div>
+
+
+                <div class="position-relative">
+                    <div class="chat-messages p-2">
+
+                        {% for date in meta['messages'] %}
+
+                            <div class="divider d-flex align-items-center mb-4">
+                                <p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
+                                    <span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
+                                </p>
+                            </div>
+
+                            {% for mess in meta['messages'][date] %}
+
+                                {% with message=mess %}
+                                    {% include 'chats_explorer/block_message.html' %}
+                                {% endwith %}
+
+                            {% endfor %}
+                            <br>
+                        {% endfor %}
+
+                    </div>
+                </div>
+
+
+
+{#                {% if meta['chats'] %}#}
+{#                    <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>#}
+{#                    <div id="heatmapweekhourall"></div>#}
+{##}
+{#                    <h4>User Chats:</h4>#}
+{#                    {% for meta_chats in meta['chats'] %}#}
+{#                        <div class="my-2">#}
+{#                            {% with meta=meta_chats  %}#}
+{#                                {% include 'chats_explorer/basic_card_chat.html' %}#}
+{#                            {% endwith %}#}
+{#                        </div>#}
+{#                    {% endfor %}#}
+{#                {% endif %}#}
+
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+  });
+
+{#d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}")#}
+{#    .then(function(data) {#}
+{#        create_heatmap_week_hour('#heatmapweekhourall', data);#}
+{#    })#}
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+
+
+
+
+</body>
+
+</html>

var/www/templates/correlation/show_relationship.html

@@ -16,7 +16,8 @@
 		<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
 		<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
 		<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
-		<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+		<script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
+        <script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
 
 		<style>
 		.icon_legend {
@@ -71,6 +72,7 @@
 
         .blured {
             filter: blur(5px);
+            max-width: 200px;
         }
 
 		.graph_panel {
@@ -127,6 +129,10 @@
 						{% include 'correlation/metadata_card_item.html' %}
 					{% endif %}
 
+                    <div class="my-2">
+                        {% include 'objects/image/block_blur_img_slider.html' %}
+                    </div>
+
 					<div class="row">
 						<div class="col-xl-10">
 
@@ -205,13 +211,49 @@
 								<div class="card-body text-center px-0 py-0">
 
 									<ul class="list-group">
-                                        <li class="list-group-item list-group-item-info">Relationship</li>
+                                        <li class="list-group-item list-group-item-info">Relationships</li>
                                         <form action="{{ url_for('correlation.show_relationship') }}" method="post">
 
+                                            <li class="list-group-item text-left">
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_from_Check" name="relationship_forwarded_from_Check" {%if "forwarded_from" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_forwarded_from_Check">Forwarded From</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_to_Check" name="relationship_forwarded_to_Check" {%if "forwarded_to" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_forwarded_to_Check">Forwarded To</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_in_Check" name="relationship_in_Check" {%if "in" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_in_Check">In</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="relationship_mention_Check">Mention</label>
+                                                </div>
+                                            </li>
+
                                             <input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
                                             <input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
                                             <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
 
+                                            <li class="list-group-item list-group-item-info">Objects Types</li>
+
+                                            <li class="list-group-item text-left">
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="chat_Check">Chat</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="message_Check">Message</label>
+                                                </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="user-account_Check">User-Account</label>
+                                                </div>
+                                            </li>
+
 {#                                        <li class="list-group-item text-left">#}
 {#                                                <div class="form-check">#}
 {#                                                    <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
@@ -313,10 +355,25 @@
                         </div>
                     </div>
 
+                    <h3>Forwards</h3>
+                    <div id="chart_test" style="max-width: 900px"></div>
+
+                    <h3>Mentions</h3>
+                    <div id="chart_mentions" style="max-width: 900px"></div>
+
+
+                    <table id="table_graph_node_objects">
+                    </table>
+
+                    <div id="timeline"></div>
+
 				</div>
 			</div>
 		</div>
 
+
+
+{% include 'objects/tooltip_ail_objects.html' %}
 <script>
 
 var all_graph = {};
@@ -324,8 +381,20 @@ $(document).ready(function(){
         $("#incomplete_graph").hide();
 		$("#page-Decoded").addClass("active");
 
-		all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
+		all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
 		all_graph.onResize();
+
+        let url = "{{ url_for('correlation.relationships_chord_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
+        d3.json(url).then(function(data) {
+            create_directed_chord_diagram('#chart_test', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
+        });
+
+        let url2 = "{{ url_for('correlation.relationships_chord_mentions_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
+        d3.json(url2).then(function(data) {
+            create_directed_chord_diagram('#chart_mentions', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
+        });
+
+
 });
 
 const blur_slider_correlation = $('#blur-slider-correlation');
@@ -553,30 +622,41 @@ d3.json(url)
 
     d3.select("body").on("keypress", keypressed)
 
+    let table_obj = document.getElementById("table_graph_node_objects")
+    for (let i=0; i<data.nodes.length; i++) {
+        let newRow = table_obj.insertRow();
+        let newCell = newRow.insertCell();
+        let newText = document.createTextNode(data.nodes[i].id);
+        newCell.appendChild(newText);
 
+        //console.log(data.nodes[i])
+
+    }
+
+    //// --------------------------------------------------------------------------------------------------------
 
 })
 .catch(function(error) {
 				$("#graph_loading").remove()
 				svg_node.remove();
 				d3.select("#graph").append("div")
-													 .text(error);
+                    .text(error);
 		});
 }
 
-function zoomed() {
+function zoomed(event, d) {
-	container_graph.attr("transform", d3.event.transform);
+	container_graph.attr("transform", event.transform);
 }
 
-function doubleclick (d) {
+function doubleclick (event, d) {
 	window.open(d.url, '_blank');
 }
 
-function keypressed () {
+function keypressed (event, d) {
-    //console.log(d3.event.keyCode)
+    //console.log(event.keyCode)
     //console.log(currentObject.id)
     // hide node, H or h key
-    if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
+    if ((event.keyCode === 72 || event.keyCode === 104) && currentObject) {
         window.location.href = correl_link + "&hide=" + currentObject.id
     }
 
@@ -586,29 +666,29 @@ function click (d) {
 console.log('clicked')
 }
 
-function drag_start(d) {
+function drag_start(event, d) {
-if (!d3.event.active) simulation.alphaTarget(0.3).restart();
+if (!event.active) simulation.alphaTarget(0.3).restart();
 d.fx = d.x;
 d.fy = d.y;
 }
 
-function dragged(d) {
+function dragged(event, d) {
-d.fx = d3.event.x;
+d.fx = event.x;
-d.fy = d3.event.y;
+d.fy = event.y;
 }
 
-function drag_end(d) {
+function drag_end(event, d) {
-if (!d3.event.active) simulation.alphaTarget(0);
+if (!event.active) simulation.alphaTarget(0);
 d.fx = d.x;
 d.fy = d.y;
 }
 
-function mouseovered(d) {
+function mouseovered(event, d, obj_gid, obj_label, additional_text) {
 
 currentObject = d;
 
-var d3_pageX = d3.event.pageX;
+var d3_pageX = event.pageX;
-var d3_pageY = d3.event.pageY;
+var d3_pageY = event.pageY;
 
 if (d.popover) {
 	div.html(d.popover)
@@ -621,8 +701,16 @@ if (d.popover) {
     blur_tooltip();
 } else {
 
-		var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
+		let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
-		var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
+        if (obj_label) {
+            pop_header = pop_header + sanitize_text(obj_label)
+        } else if (obj_gid) {
+            pop_header = pop_header + sanitize_text(obj_gid)
+        } else {
+            pop_header = pop_header + sanitize_text(d.text)
+        }
+        pop_header = pop_header + "</h6></div>"
+		let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
 
 		div.html(pop_header + spinner)
 			.style("left", (d3_pageX) + "px")
@@ -632,9 +720,16 @@ if (d.popover) {
 			.duration(200)
 			.style("opacity", 1);
 
-		$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
+        let description_url = "{{ url_for('correlation.get_description') }}?object_id="
+        if (obj_gid) {
+            description_url = description_url + obj_gid
+        } else {
+            description_url = description_url + d.id
+        }
+
+		$.getJSON(description_url,
 				function(data){
-					var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
+					let desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
 					Object.keys(data).forEach(function(key) {
 						if (key=="status") {
 							desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
@@ -650,9 +745,16 @@ if (d.popover) {
 								desc = desc + "fa-times-circle\"></i>DOWN"
 							}
 							desc = desc + "</div></dd>"
-						} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
+						} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
-							desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+							if (data[key]) {
-						}
+                                if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
+                                    let date = sanitize_text(data[key])
+                                    desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
+                                } else {
+							        desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+                                }
+                            }
+                        }
 					});
 					desc = desc + "</dl>"
 
@@ -674,6 +776,9 @@ if (d.popover) {
                             desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
                         }
 					}
+                    if (additional_text) {
+                        desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
+                    }
 
 					desc = desc + "</div></div>"
 					div.html(desc)
@@ -697,7 +802,6 @@ if (d.popover) {
 
 function mouseouted() {
     currentObject = null;
-
 	div.transition()
 		.duration(500)
 		.style("opacity", 0);
@@ -712,8 +816,18 @@ all_graph.onResize = function () {
 }
 
 window.all_graph = all_graph;
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+/*
+*/
+
+
+
 </script>
 
-        </body>
+</body>
 
 </html>

var/www/templates/crawler/crawler_splash/last_crawled.html

@@ -95,6 +95,10 @@
                 <div id="barchart_type_month"></div>
                 <div class="text-center" id="pie_chart_month"></div>
 
+                <h3>Previous Month Stats:</h3>
+                <div id="barchart_type_month"></div>
+                <div class="text-center" id="barchart_type_previous_month"></div>
+
 			</div>
 
 		</div>
@@ -156,6 +160,13 @@ $(document).ready(function(){
         }
     );
 
+    $.getJSON("{{ url_for('crawler_splash.crawlers_last_domains_previous_month_json') }}?type={{type}}",
+        function (data) {
+                let div_width = $("#barchart_type_previous_month").width();
+                barchart_stack("barchart_type_previous_month", data, {"width": div_width});
+        }
+    );
+
 });
 
 function toggle_sidebar(){

var/www/templates/hunter/tracker_add.html

@@ -75,7 +75,7 @@
 {#                                                        <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
 {#                                                    </div>#}
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" {% if not dict_tracker['filters'] or 'decoded' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Content that has been decoded from an encoded format, such as base64"></i></label>
                                                     </div>
 {#                                                    <div class="custom-control custom-switch mt-1">#}
@@ -83,7 +83,7 @@
 {#                                                        <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
 {#                                                    </div>#}
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" {% if not dict_tracker['filters'] or 'item' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Text that has been processed by AIL. It can include various types of extracted information"></i></label>
                                                     </div>
                                                     <div class="card border-dark mb-4" id="sources_item_div">
@@ -100,14 +100,14 @@
 
 
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" {% if not dict_tracker['filters'] or 'pgp' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="PGP key/block metadata"></i></label>
                                                     </div>
                                                     <div class="card border-dark mb-4" id="sources_pgp_div">
                                                         <div class="card-body">
                                                             <h6>Filter PGP by subtype:</h6>
                                                             <div class="custom-control custom-switch mt-1">
-                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
+                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'name' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
                                                                 <label class="custom-control-label" for="filter_pgp_name">
                                                                     <svg height="26" width="26">
                                                                         <g class="nodes">
@@ -119,7 +119,7 @@
                                                                 </label>
                                                             </div>
                                                             <div class="custom-control custom-switch mt-1">
-                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
+                                                                <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'mail' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
                                                                 <label class="custom-control-label" for="filter_pgp_mail">
                                                                     <svg height="26" width="26">
                                                                         <g class="nodes">
@@ -133,15 +133,15 @@
                                                         </div>
                                                     </div>
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" {% if not dict_tracker['filters'] or 'title' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="title_obj"><i class="fas fa-heading"></i>&nbsp;Title <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Title that has been extracted from a HTML page"></i></label>
                                                     </div>
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" {% if not dict_tracker['filters'] or 'message' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i>&nbsp;Message <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Messages from Chats"></i></label>
                                                     </div>
                                                     <div class="custom-control custom-switch mt-1">
-                                                        <input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" checked="">
+                                                        <input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" {% if not dict_tracker['filters'] or 'ocr' in dict_tracker['filters'] %}checked=""{% endif %}>
                                                         <label class="custom-control-label" for="ocr_obj"><i class="fas fa-comment-dots"></i>&nbsp;OCR <i class="fas fa-expand text-info" data-toggle="tooltip" data-placement="right" title="Text extracted from Images"></i></label>
                                                     </div>
 
@@ -310,6 +310,9 @@ $(document).ready(function(){
 		emptyText: 'Item Sources to track (ALL IF EMPTY)',
 		});
 
+    item_source_input_controller();
+    pgp_source_input_controller();
+
 	$('#tracker_type').on('change', function() {
 		var tracker_type = this.value;
 		if (tracker_type=="word") {

var/www/templates/objects/tooltip_ail_objects.html

@@ -0,0 +1,136 @@
+<script>
+let tooltip_ail_obj = d3.select("body").append("div")
+    //.attr("class", "tooltip_graph")
+    .attr("id", "tooltip_graph")
+    .style("opacity", 0)
+    .style("position", "absolute")
+    .style("text-align", "center")
+    .style("padding", "2px")
+    .style("font", "12px sans-serif")
+    .style("background", "#ebf4fb")
+    .style("border", "2px solid #b7ddf2")
+    .style("border-radius", "8px")
+    .style("pointer-events", "none")
+    .style("color", "#000000");
+
+function mouseover_tooltip_ail_obj(event, d, obj_gid, obj_label, additional_text) { /// div var/const tooltip     tooltip_ail_obj
+
+    let d3_pageX = event.pageX;
+    let d3_pageY = event.pageY;
+
+    if (d.popover) {
+        tooltip_ail_obj.html(d.popover)
+            .style("left", (d3_pageX) + "px")
+            .style("top", (d3_pageY - 28) + "px");
+
+        tooltip_ail_obj.transition()
+            .duration(200)
+            .style("opacity", 1);
+        blur_images();
+    } else {
+
+		let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
+        if (obj_label) {
+            pop_header = pop_header + sanitize_text(obj_label)
+        } else if (obj_gid) {
+            pop_header = pop_header + sanitize_text(obj_gid)
+        } else {
+            pop_header = pop_header + sanitize_text(d.text)
+        }
+        pop_header = pop_header + "</h6></div>"
+		let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
+
+		tooltip_ail_obj.html(pop_header + spinner)
+			.style("left", (d3_pageX) + "px")
+			.style("top", (d3_pageY - 28) + "px");
+
+		tooltip_ail_obj.transition()
+			.duration(200)
+			.style("opacity", 1);
+
+        let description_url = "{{ url_for('correlation.get_description') }}?object_id="
+        if (obj_gid) {
+            description_url = description_url + obj_gid
+        } else {
+            description_url = description_url + d.id
+        }
+
+        let desc
+        $.getJSON(description_url, function(data){
+            desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
+            Object.keys(data).forEach(function(key) {
+                if (key==="status") {
+                    desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
+                    if (data["status"]) {
+                        desc = desc + "Green"
+                    } else {
+                        desc = desc + "Red"
+                    }
+                    desc = desc + ";\"><i class=\"fas "
+                    if (data["status"]) {
+                        desc = desc + "fa-check-circle\"></i>UP"
+                    } else {
+                        desc = desc + "fa-times-circle\"></i>DOWN"
+                    }
+                    desc = desc + "</div></dd>"
+                } else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
+                    if (data[key]) {
+                        if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
+                            let date = sanitize_text(data[key])
+                            desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
+                        } else {
+                            desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+                        }
+                    }
+                }
+            });
+            desc = desc + "</dl>"
+
+            if (data["tags"]) {
+                data["tags"].forEach(function(tag) {
+                    desc = desc + "<span class=\"badge badge-warning\">"+ sanitize_text(tag) +"</span>";
+                });
+            }
+
+            if (data["img"]) {
+                if (data["tags_safe"]) {
+                    if (data["type"] === "screenshot") {
+                        desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
+                    } else {
+                        desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
+                    }
+                    desc = desc + data["img"] +" class=\"object_image\" id=\"tooltip_screenshot_correlation\" style=\"filter: blur(5px);max-width: 200px;\"/>";
+                } else {
+                    desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
+                }
+            }
+            if (additional_text) {
+                desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
+            }
+
+            desc = desc + "</div></div>"
+            tooltip_ail_obj.html(desc)
+                .style("left", (d3_pageX) + "px")
+                .style("top", (d3_pageY - 28) + "px");
+            d.popover = desc
+
+            if (data["img"]) {
+                blur_images();
+            }
+
+        })
+            .fail(function(error) {
+                desc = pop_header + "<div class=\"card-body bg-dark pt-0\"><i class=\"fas fa-3x fa-times text-danger\"></i>"+ error.statusText +"</div>"
+                tooltip_ail_obj.html(desc)
+                    .style("left", (d3_pageX) + "px")
+                    .style("top", (d3_pageY - 28) + "px");
+            });
+    }
+}
+
+function mouseout_tooltip_ail_obj(event, d) {
+    tooltip_ail_obj.transition()
+        .duration(500)
+        .style("opacity", 0);
+}
+</script>

var/www/update_thirdparty.sh

@@ -9,7 +9,9 @@ wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.
 
 BOOTSTRAP_VERSION='4.2.1'
 FONT_AWESOME_VERSION='5.7.1'
+
 D3_JS_VERSION='5.16.0'
+wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
 
 rm -rf temp
 mkdir temp