chg: [chat + user-account] correlations + usernames timeline

bin/crawlers/Crawler.py

@@ -282,7 +282,7 @@ class Crawler(AbstractModule):
             title_content = crawlers.extract_title_from_html(entries['html'])
             if title_content:
                 title = Titles.create_title(title_content)
-                title.add(item.get_date(), item_id)
+                title.add(item.get_date(), item)
 
             # SCREENSHOT
             if self.screenshot:
@@ -306,11 +306,11 @@ class Crawler(AbstractModule):
                     for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']):
                         print(cookie_name)
                         cookie = CookiesNames.create(cookie_name)
-                        cookie.add(self.date.replace('/', ''), self.domain.id)
+                        cookie.add(self.date.replace('/', ''), self.domain)
                     for etag_content in crawlers.extract_etag_from_har(entries['har']):
                         print(etag_content)
                         etag = Etags.create(etag_content)
-                        etag.add(self.date.replace('/', ''), self.domain.id)
+                        etag.add(self.date.replace('/', ''), self.domain)
                     crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', ''))
 
         # Next Children

bin/importer/feeders/Jabber.py

@@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from importer.feeders.Default import DefaultFeeder
 from lib.objects.Usernames import Username
-from lib import item_basic
+from lib.objects.Items import Item
 
 
 class JabberFeeder(DefaultFeeder):
@@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder):
         self.item_id = f'{item_id}.gz'
         return self.item_id
 
-    def process_meta(self):
+    def process_meta(self): # TODO replace me by message
         """
         Process JSON meta field.
         """
@@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder):
         # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ##############################################
         to = str(self.json_data['meta']['jabber:to'])
         fr = str(self.json_data['meta']['jabber:from'])
-        date = item_basic.get_item_date(item_id)
+
+        item = Item(self.item_id)
+        date = item.get_date()
 
         user_to = Username(to, 'jabber')
         user_fr = Username(fr, 'jabber')
-        user_to.add(date, self.item_id)
+        user_to.add(date, item)
-        user_fr.add(date, self.item_id)
+        user_fr.add(date, item)
         return None

bin/importer/feeders/Telegram.py

@@ -21,7 +21,6 @@ from lib.objects.Chats import Chat
 from lib.objects import Messages
 from lib.objects import UsersAccount
 from lib.objects.Usernames import Username
-from lib import item_basic
 
 import base64
 import io
@@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder):
         #     date = datetime.date.today().strftime("%Y/%m/%d")
         chat_id = str(self.json_data['meta']['chat']['id'])
         message_id = str(self.json_data['meta']['id'])
-        self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
+        self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)  # TODO rename self.item_id
         return self.item_id
 
     def process_meta(self):
@@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder):
         meta = self.json_data['meta']
         mess_id = self.json_data['meta']['id']
         if meta.get('reply_to'):
-            reply_to_id = meta['reply_to']
+            reply_to_id = meta['reply_to']['id']
         else:
             reply_to_id = None
 
@@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder):
         date = datetime.datetime.fromtimestamp(timestamp)
         date = date.strftime('%Y%m%d')
 
+        if self.json_data.get('translation'):
+            translation = self.json_data['translation']
+        else:
+            translation = None
+        decoded = base64.standard_b64decode(self.json_data['data'])
+        content = gunzip_bytes_obj(decoded)
+        message = Messages.create(self.item_id, content, translation=translation)
+
         if meta.get('chat'):
             chat = Chat(meta['chat']['id'], 'telegram')
 
-            if meta['chat'].get('username'):  # SAVE USERNAME
+            if meta['chat'].get('username'):
-                chat_username = meta['chat']['username']
+                chat_username = Username(meta['chat']['username'], 'telegram')
+                chat.update_username_timeline(chat_username.get_global_id(), timestamp)
 
             # Chat---Message
-            chat.add(date, self.item_id)  # TODO modify to accept file objects
+            chat.add(date)
-            # message meta ????? who is the user if two user ????
+            chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
-
-            if self.json_data.get('translation'):
-                translation = self.json_data['translation']
-            else:
-                translation = None
-            decoded = base64.standard_b64decode(self.json_data['data'])
-            content = gunzip_bytes_obj(decoded)
-            Messages.create(self.item_id, content, translation=translation)
-
-            chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id)
         else:
             chat = None
 
@@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder):
             user_id = meta['sender']['id']
             user_account = UsersAccount.UserAccount(user_id, 'telegram')
             # UserAccount---Message
-            user_account.add(date, self.item_id)
+            user_account.add(date, obj=message)
             # UserAccount---Chat
             user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
 
@@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder):
 
             if meta['sender'].get('username'):
                 username = Username(meta['sender']['username'], 'telegram')
+                # TODO timeline or/and correlation ????
                 user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
-                # TODO Update user_account<--->username timeline
+                user_account.update_username_timeline(username.get_global_id(), timestamp)
 
                 # Username---Message
-                username.add(date, self.item_id) # TODO ####################################################################
+                username.add(date)  # TODO # correlation message ???
 
-                if chat:
+                # if chat: # TODO Chat---Username correlation ???
-                    # Chat---Username
+                #     # Chat---Username
-                    chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+                #     chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
 
         # if meta.get('fwd_from'):
             # if meta['fwd_from'].get('post_author') # user first name
 
         # TODO reply threads ????
+        # message edit ????
 
 
         return None

bin/importer/feeders/Twitter.py

@@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from importer.feeders.Default import DefaultFeeder
 from lib.objects.Usernames import Username
-from lib import item_basic
+from lib.objects.Items import Item
 
 class TwitterFeeder(DefaultFeeder):
 
@@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder):
         '''
         # tweet_id = str(self.json_data['meta']['twitter:tweet_id'])
         # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################
-
+        item = Item(self.item_id)
-        date = item_basic.get_item_date(self.item_id)
+        date = item.get_date()
         user = str(self.json_data['meta']['twitter:id'])
         username = Username(user, 'twitter')
-        username.add(date, item_id)
+        username.add(date, item)
         return None

bin/lib/correlations_engine.py

@@ -41,20 +41,22 @@ config_loader = None
 ##################################
 
 CORRELATION_TYPES_BY_OBJ = {
-    "chat": ["item", "username"],  # item ???
+    "chat": ["user-account"],  # message or direct correlation like cve, bitcoin, ... ???
     "cookie-name": ["domain"],
-    "cryptocurrency": ["domain", "item"],
+    "cryptocurrency": ["domain", "item", "message"],
-    "cve": ["domain", "item"],
+    "cve": ["domain", "item", "message"],
-    "decoded": ["domain", "item"],
+    "decoded": ["domain", "item", "message"],
     "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
     "etag": ["domain"],
     "favicon": ["domain", "item"],  # TODO Decoded
     "hhhash": ["domain"],
-    "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
+    "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],  # chat ???
-    "pgp": ["domain", "item"],
+    "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"],  # chat ??
+    "pgp": ["domain", "item", "message"],
     "screenshot": ["domain", "item"],
     "title": ["domain", "item"],
-    "username": ["chat", "domain", "item"],
+    "user-account": ["chat", "message"],
+    "username": ["domain", "item", "message"],  # TODO chat-user/account
 }
 
 def get_obj_correl_types(obj_type):

bin/lib/crawlers.py

@@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name():
         for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)):
             print(domain, date, cookie_name)
             cookie = CookiesNames.create(cookie_name)
-            cookie.add(date, domain)
+            cookie.add(date, Domain(domain))
 
 def extract_etag_from_har(har):  # TODO check response url
     etags = set()
@@ -365,7 +365,7 @@ def _reprocess_all_hars_etag():
         for etag_content in extract_etag_from_har(get_har_content(har_id)):
             print(domain, date, etag_content)
             etag = Etags.create(etag_content)
-            etag.add(date, domain)
+            etag.add(date, Domain(domain))
 
 def extract_hhhash_by_id(har_id, domain, date):
     return extract_hhhash(get_har_content(har_id), domain, date)
@@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date):
 
                         # -----
                         obj = HHHashs.create(hhhash_header, hhhash)
-                        obj.add(date, domain)
+                        obj.add(date, Domain(domain))
 
                     hhhashs.add(hhhash)
                     urls.add(url)

bin/lib/objects/Chats.py

@@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
 from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 from lib.data_retention_engine import update_obj_date
 from lib.objects import ail_objects
-from lib import item_basic
+from lib.timeline_engine import Timeline
 
 from lib.correlations_engine import get_correlation_by_correl_type
 
@@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
             users.add(account[1:])
         return users
 
+    def _get_timeline_username(self):
+        return Timeline(self.get_global_id(), 'username')
+
+    def get_username(self):
+        return self._get_timeline_username().get_last_obj_id()
+
+    def get_usernames(self):
+        return self._get_timeline_username().get_objs_ids()
+
+    def update_username_timeline(self, username_global_id, timestamp):
+        self._get_timeline_username().add_timestamp(timestamp, username_global_id)
+
 
     # def get_last_message_id(self):
     #
@@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
 
     def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
         obj = ail_objects.get_obj_from_global_id(obj_global_id)
-        mess_dict = obj.get_meta(options={'content', 'link', 'parent'})
+        mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
         if mess_dict.get('parent') and parent:
             mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
-        mess_dict['username'] = {}
+        if mess_dict.get('user-account'):
-        user = obj.get_correlation('username').get('username')
+            user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
-        if user:
+            mess_dict['user-account'] = {}
-            subtype, user = user.pop().split(':', 1)
+            mess_dict['user-account']['type'] = user_account.get_type()
-            mess_dict['username']['type'] = 'telegram'
+            mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
-            mess_dict['username']['subtype'] = subtype
+            mess_dict['user-account']['id'] = user_account.get_id()
-            mess_dict['username']['id'] = user
+            username = user_account.get_username()
+            if username:
+                username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
+            mess_dict['user-account']['username'] = username  # TODO get username at the given timestamp ???
         else:
-            mess_dict['username']['id'] = 'UNKNOWN'
+            mess_dict['user-account']['id'] = 'UNKNOWN'
 
         if not mess_datetime:
             obj_mess_id = self._get_message_timestamp(obj_global_id)

bin/lib/objects/Messages.py

@@ -27,7 +27,7 @@ from flask import url_for
 config_loader = ConfigLoader()
 r_cache = config_loader.get_redis_conn("Redis_Cache")
 r_object = config_loader.get_db_conn("Kvrocks_Objects")
-r_content = config_loader.get_db_conn("Kvrocks_Content")
+# r_content = config_loader.get_db_conn("Kvrocks_Content")
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None
 
@@ -61,7 +61,7 @@ class Message(AbstractObject):
         """
         Returns source/feeder name
         """
-        l_source = self.id.split('/')[:-4]
+        l_source = self.id.split('/')[:-2]
         return os.path.join(*l_source)
 
     def get_basename(self):
@@ -79,7 +79,7 @@ class Message(AbstractObject):
 
     def get_date(self):
         timestamp = self.get_timestamp()
-        return datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
+        return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
 
     def get_timestamp(self):
         dirs = self.id.split('/')
@@ -92,11 +92,16 @@ class Message(AbstractObject):
         return message_id
 
     def get_chat_id(self):  # TODO optimize -> use me to tag Chat
-        chat_id =  self.get_basename().rsplit('_', 1)[0]
+        chat_id = self.get_basename().rsplit('_', 1)[0]
         # if chat_id.endswith('.gz'):
         #     chat_id = chat_id[:-3]
         return chat_id
 
+    def get_user_account(self):
+        user_account = self.get_correlation('user-account')
+        if user_account.get('user-account'):
+            return f'user-account:{user_account["user-account"].pop()}'
+
     # Update value on import
     # reply to -> parent ?
     # reply/comment - > children ?
@@ -139,7 +144,7 @@ class Message(AbstractObject):
         return url
 
     def get_svg_icon(self):
-        return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5}
+        return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5}
 
     def get_misp_object(self):  # TODO
         obj = MISPObject('instant-message', standalone=True)
@@ -181,6 +186,8 @@ class Message(AbstractObject):
             meta['investigations'] = self.get_investigations()
         if 'link' in options:
             meta['link'] = self.get_link(flask_context=True)
+        if 'user-account' in options:
+            meta['user-account'] = self.get_user_account()
 
         # meta['encoding'] = None
         return meta
@@ -238,7 +245,7 @@ class Message(AbstractObject):
 
     def create(self, content, translation, tags):
         self._set_field('content', content)
-        r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
+        # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
         if translation:
             self._set_translation(translation)
         for tag in tags:

bin/lib/objects/UsersAccount.py

@@ -3,7 +3,7 @@
 
 import os
 import sys
-import re
+# import re
 
 from flask import url_for
 from pymisp import MISPObject
@@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib import ail_core
 from lib.ConfigLoader import ConfigLoader
 from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+from lib.timeline_engine import Timeline
 
 config_loader = ConfigLoader()
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject):
     def set_phone(self, phone):
         return self._set_field('phone', phone)
 
-    # TODO REWRITE ADD FUNCTION
+    def _get_timeline_username(self):
+        return Timeline(self.get_global_id(), 'username')
 
     def get_username(self):
-        return ''
+        return self._get_timeline_username().get_last_obj_id()
 
     def get_usernames(self):
-        usernames = []
+        return self._get_timeline_username().get_objs_ids()
-        # TODO TIMELINE
+
-        return usernames
+    def update_username_timeline(self, username_global_id, timestamp):
+        self._get_timeline_username().add_timestamp(timestamp, username_global_id)
 
     def get_meta(self, options=set()):
         meta = self._get_meta(options=options)

bin/lib/objects/abstract_object.py

@@ -65,12 +65,14 @@ class AbstractObject(ABC):
     def get_global_id(self):
         return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}'
 
-    def get_default_meta(self, tags=False):
+    def get_default_meta(self, tags=False, link=False):
         dict_meta = {'id': self.get_id(),
                      'type': self.get_type(),
                      'subtype': self.get_subtype(r_str=True)}
         if tags:
             dict_meta['tags'] = self.get_tags()
+        if link:
+            dict_meta['link'] = self.get_link()
         return dict_meta
 
     def _get_field(self, field):

bin/lib/objects/abstract_subtype_object.py

@@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC):
 #
 #
 
-    def add(self, date, item_id):
+    def add(self, date, obj=None):
         self.update_daterange(date)
         update_obj_date(date, self.type, self.subtype)
         # daily
@@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC):
         #######################################################################
         #######################################################################
 
-        # Correlations
+        if obj:
-        self.add_correlation('item', '', item_id)
+            # Correlations
-        # domain
+            self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id())
-        if is_crawled(item_id):
-            domain = get_item_domain(item_id)
-            self.add_correlation('domain', '', domain)
 
+            if obj.type == 'item': # TODO same for message->chat ???
+                item_id = obj.get_id()
+                # domain
+                if is_crawled(item_id):
+                    domain = get_item_domain(item_id)
+                    self.add_correlation('domain', '', domain)
 
     # TODO:ADD objects + Stats
     def create(self, first_seen, last_seen):
         self.set_first_seen(first_seen)
         self.set_last_seen(last_seen)
 
-
     def _delete(self):
         pass
 

bin/lib/objects/ail_objects.py

@@ -23,9 +23,11 @@ from lib.objects import Etags
 from lib.objects.Favicons import Favicon
 from lib.objects import HHHashs
 from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
+from lib.objects.Messages import Message
 from lib.objects import Pgps
 from lib.objects.Screenshots import Screenshot
 from lib.objects import Titles
+from lib.objects.UsersAccount import UserAccount
 from lib.objects import Usernames
 
 config_loader = ConfigLoader()
@@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id):
         return Favicon(id)
     elif obj_type == 'hhhash':
         return HHHashs.HHHash(id)
+    elif obj_type == 'message':
+        return Message(id)
     elif obj_type == 'screenshot':
         return Screenshot(id)
     elif obj_type == 'cryptocurrency':
@@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id):
         return Pgps.Pgp(id, subtype)
     elif obj_type == 'title':
         return Titles.Title(id)
+    elif obj_type == 'user-account':
+        return UserAccount(id, subtype)
     elif obj_type == 'username':
         return Usernames.Username(id, subtype)
 

bin/lib/timeline_engine.py

@@ -46,112 +46,167 @@ config_loader = None
 #             return []
 #     return correl_types
 
-# TODO rename all function + add missing parameters
+class Timeline:
 
-def get_bloc_obj_global_id(bloc):
+    def __init__(self, global_id, name):
-    return r_meta.hget('hset:key', bloc)
+        self.id = global_id
+        self.name = name
 
-def set_bloc_obj_global_id(bloc, global_id):
+    def _get_block_obj_global_id(self, block):
-    return r_meta.hset('hset:key', bloc, global_id)
+        return r_meta.hget(f'block:{self.id}:{self.name}', block)
 
-def get_bloc_timestamp(bloc, position):
+    def _set_block_obj_global_id(self, block, global_id):
-    return r_meta.zscore('key', f'{position}:{bloc}')
+        return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id)
 
-def add_bloc(global_id, timestamp, end=None):
+    def _get_block_timestamp(self, block, position):
-    if end:
+        return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}')
-        timestamp_end = end
-    else:
-        timestamp_end = timestamp
-    new_bloc = str(uuid4())
-    r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
-    set_bloc_obj_global_id(new_bloc, global_id)
-    return new_bloc
 
-def _update_bloc(bloc, position, timestamp):
+    def _get_nearest_bloc_inf(self, timestamp):
-    r_meta.zadd('key', {f'{position}:{bloc}': timestamp})
+        inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True)
+        if inf:
+            inf, score = inf[0]
+            if inf.startswith('end'):
+                inf_key = f'start:{inf[4:]}'
+                inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key)
+                if inf_score == score:
+                    inf = inf_key
+            return inf
+        else:
+            return None
 
-# score = timestamp
+    def _get_nearest_bloc_sup(self, timestamp):
-def get_nearest_bloc_inf(timestamp):
+        sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True)
-    return r_meta.zrevrangebyscore('key', timestamp, 0, num=1)
+        if sup:
+            sup, score = sup[0]
+            if sup.startswith('start'):
+                sup_key = f'end:{sup[6:]}'
+                sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key)
+                if score == sup_score:
+                    sup = sup_key
+            return sup
+        else:
+            return None
 
-def get_nearest_bloc_sup(timestamp):
+    def get_first_obj_id(self):
-    return r_meta.zrangebyscore('key', timestamp, 0, num=1)
+        first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0)
+        if first:  # start:block
+            first = first[0]
+            if first.startswith('start:'):
+                first = first[6:]
+            else:
+                first = first[4:]
+            return self._get_block_obj_global_id(first)
 
-#######################################################################################
+    def get_last_obj_id(self):
+        last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0)
+        if last:  # end:block
+            last = last[0]
+            if last.startswith('end:'):
+                last = last[4:]
+            else:
+                last = last[6:]
+            return self._get_block_obj_global_id(last)
 
-def add_timestamp(timestamp, obj_global_id):
+    def get_objs_ids(self):
-    inf = get_nearest_bloc_inf(timestamp)
+        objs = set()
-    sup = get_nearest_bloc_sup(timestamp)
+        for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1):
-    if not inf and not sup:
+            if block:
-        # create new bloc
+                if block.startswith('start:'):
-        new_bloc = add_bloc(obj_global_id, timestamp)
+                    objs.add(self._get_block_obj_global_id(block[6:]))
+        return objs
+
+    # def get_objs_ids(self):
+    #     objs = {}
+    #     last_obj_id = None
+    #     for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True):
+    #         if block:
+    #             if block.startswith('start:'):
+    #                 last_obj_id = self._get_block_obj_global_id(block[6:])
+    #                 objs[last_obj_id] = {'first_seen': timestamp}
+    #             else:
+    #                 objs[last_obj_id]['last_seen'] = timestamp
+    #     return objs
+
+    def _update_bloc(self, block, position, timestamp):
+        r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp})
+
+    def _add_bloc(self, obj_global_id, timestamp, end=None):
+        if end:
+            timestamp_end = end
+        else:
+            timestamp_end = timestamp
+        new_bloc = str(uuid4())
+        r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
+        self._set_block_obj_global_id(new_bloc, obj_global_id)
         return new_bloc
-    # timestamp < first_seen
+
-    elif not inf:
+    def add_timestamp(self, timestamp, obj_global_id):
-        sup_pos, sup_id = inf.split(':')
+        inf = self._get_nearest_bloc_inf(timestamp)
-        sup_obj = get_bloc_obj_global_id(sup_pos)
+        sup = self._get_nearest_bloc_sup(timestamp)
-        if sup_obj == obj_global_id:
+        if not inf and not sup:
-            _update_bloc(sup_id, 'start', timestamp)
+            # create new bloc
-        # create new bloc
+            new_bloc = self._add_bloc(obj_global_id, timestamp)
-        else:
-            new_bloc = add_bloc(obj_global_id, timestamp)
             return new_bloc
-
+        # timestamp < first_seen
-    # timestamp > first_seen
+        elif not inf:
-    elif not sup:
+            sup_pos, sup_id = sup.split(':')
-        inf_pos, inf_id = inf.split(':')
+            sup_obj = self._get_block_obj_global_id(sup_id)
-        inf_obj = get_bloc_obj_global_id(inf_id)
+            if sup_obj == obj_global_id:
-        if inf_obj == obj_global_id:
+                self._update_bloc(sup_id, 'start', timestamp)
-            _update_bloc(inf_id, 'end', timestamp)
+            # create new bloc
-        # create new bloc
+            else:
-        else:
+                new_bloc = self._add_bloc(obj_global_id, timestamp)
-            new_bloc = add_bloc(obj_global_id, timestamp)
-            return new_bloc
-
-    else:
-        inf_pos, inf_id = inf.split(':')
-        sup_pos, sup_id = inf.split(':')
-        inf_obj = get_bloc_obj_global_id(inf_id)
-
-        if inf_id == sup_id:
-            # reduce bloc + create two new bloc
-            if obj_global_id != inf_obj:
-                # get end timestamp
-                sup_timestamp = get_bloc_timestamp(sup_id, 'end')
-                # reduce original bloc
-                _update_bloc(inf_id, 'end', timestamp - 1)
-                # Insert new bloc
-                new_bloc = add_bloc(obj_global_id, timestamp)
-                # Recreate end of the first bloc by a new bloc
-                add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
                 return new_bloc
 
-            # timestamp in existing bloc
+        # timestamp > first_seen
-            else:
+        elif not sup:
-                return inf_id
+            inf_pos, inf_id = inf.split(':')
-
+            inf_obj = self._get_block_obj_global_id(inf_id)
-        # different blocs: expend sup/inf bloc or create a new bloc if
+            if inf_obj == obj_global_id:
-        elif inf_pos == 'end' and sup_pos == 'start':
+                self._update_bloc(inf_id, 'end', timestamp)
-            # Extend inf bloc
-            if obj_global_id == inf_obj:
-                _update_bloc(inf_id, 'end', timestamp)
-                return inf_id
-
-            sup_obj = get_bloc_obj_global_id(sup_pos)
-            # Extend sup bloc
-            if obj_global_id == sup_obj:
-                _update_bloc(sup_id, 'start', timestamp)
-                return sup_id
-
             # create new bloc
-            new_bloc = add_bloc(obj_global_id, timestamp)
+            else:
-            return new_bloc
+                new_bloc = self._add_bloc(obj_global_id, timestamp)
+                return new_bloc
 
-        # inf_pos == 'start' and sup_pos == 'end'
+        else:
-        # else raise error ???
+            inf_pos, inf_id = inf.split(':')
+            sup_pos, sup_id = sup.split(':')
+            inf_obj = self._get_block_obj_global_id(inf_id)
 
+            if inf_id == sup_id:
+                # reduce bloc + create two new bloc
+                if obj_global_id != inf_obj:
+                    # get end timestamp
+                    sup_timestamp = self._get_block_timestamp(sup_id, 'end')
+                    # reduce original bloc
+                    self._update_bloc(inf_id, 'end', timestamp - 1)
+                    # Insert new bloc
+                    new_bloc = self._add_bloc(obj_global_id, timestamp)
+                    # Recreate end of the first bloc by a new bloc
+                    self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
+                    return new_bloc
 
+                # timestamp in existing bloc
+                else:
+                    return inf_id
 
+            # different blocs: expend sup/inf bloc or create a new bloc if
+            elif inf_pos == 'end' and sup_pos == 'start':
+                # Extend inf bloc
+                if obj_global_id == inf_obj:
+                    self._update_bloc(inf_id, 'end', timestamp)
+                    return inf_id
 
+                sup_obj = self._get_block_obj_global_id(sup_id)
+                # Extend sup bloc
+                if obj_global_id == sup_obj:
+                    self._update_bloc(sup_id, 'start', timestamp)
+                    return sup_id
 
+                # create new bloc
+                new_bloc = self._add_bloc(obj_global_id, timestamp)
+                return new_bloc
 
+            # inf_pos == 'start' and sup_pos == 'end'
+            # else raise error ???

bin/modules/Cryptocurrencies.py

@@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC):
                     if crypto.is_valid_address():
                         # print(address)
                         is_valid_address = True
-                        crypto.add(date, item_id)
+                        crypto.add(date, item)
 
                 # Check private key
                 if is_valid_address:

bin/modules/PgpDump.py

@@ -210,18 +210,18 @@ class PgpDump(AbstractModule):
             date = item.get_date()
             for key in self.keys:
                 pgp = Pgps.Pgp(key, 'key')
-                pgp.add(date, self.item_id)
+                pgp.add(date, item)
                 print(f'    key: {key}')
             for name in self.names:
                 pgp = Pgps.Pgp(name, 'name')
-                pgp.add(date, self.item_id)
+                pgp.add(date, item)
                 print(f'    name: {name}')
                 self.tracker_term.compute(name, obj_type='pgp', subtype='name')
                 self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
                 self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
             for mail in self.mails:
                 pgp = Pgps.Pgp(mail, 'mail')
-                pgp.add(date, self.item_id)
+                pgp.add(date, item)
                 print(f'    mail: {mail}')
                 self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
                 self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')

bin/modules/Telegram.py

@@ -58,7 +58,7 @@ class Telegram(AbstractModule):
             user_id = dict_url.get('username')
             if user_id:
                 username = Username(user_id, 'telegram')
-                username.add(item_date, item.id)
+                username.add(item_date, item)
                 print(f'username: {user_id}')
             invite_hash = dict_url.get('invite_hash')
             if invite_hash:
@@ -73,7 +73,7 @@ class Telegram(AbstractModule):
             user_id = dict_url.get('username')
             if user_id:
                 username = Username(user_id, 'telegram')
-                username.add(item_date, item.id)
+                username.add(item_date, item)
                 print(f'username: {user_id}')
             invite_hash = dict_url.get('invite_hash')
             if invite_hash:

var/www/templates/objects/chat/ChatMessages.html

@@ -120,14 +120,26 @@
 
                                 <div class="chat-message-left pb-0">
                                     <div>
-                                        <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['username']['id'] }}" width="40" height="40">
+                                        <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
                                         <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
                                     </div>
                                     <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
-                                        <div class="font-weight-bold mb-1">{{ mess['username']['id'] }}</div>
+                                        <div class="font-weight-bold mb-1">
+                                            {% if mess['user-account']['username'] %}
+                                            	{{ mess['user-account']['username']['id'] }}
+                                            {% else %}
+                                            	{{ mess['user-account']['id'] }}
+                                            {% endif %}
+                                        </div>
                                         {% if mess['reply_to'] %}
                                             <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
-                                                <div class="font-weight-bold mb-1">{{ mess['reply_to']['username']['id'] }}</div>
+                                                <div class="font-weight-bold mb-1">
+                                                    {% if mess['reply_to']['user-account']['username'] %}
+                                                        {{ mess['reply_to']['user-account']['username']['id'] }}
+                                                    {% else %}
+                                                        {{ mess['reply_to']['user-account']['id'] }}
+                                                    {% endif %}
+                                                </div>
                                                 <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
                                                 {% for tag in mess['reply_to']['tags'] %}
                                                     <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>