diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 7f2c3df9..d16ad9f7 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -282,7 +282,7 @@ class Crawler(AbstractModule): title_content = crawlers.extract_title_from_html(entries['html']) if title_content: title = Titles.create_title(title_content) - title.add(item.get_date(), item_id) + title.add(item.get_date(), item) # SCREENSHOT if self.screenshot: @@ -306,11 +306,11 @@ class Crawler(AbstractModule): for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']): print(cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(self.date.replace('/', ''), self.domain.id) + cookie.add(self.date.replace('/', ''), self.domain) for etag_content in crawlers.extract_etag_from_har(entries['har']): print(etag_content) etag = Etags.create(etag_content) - etag.add(self.date.replace('/', ''), self.domain.id) + etag.add(self.date.replace('/', ''), self.domain) crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', '')) # Next Children diff --git a/bin/importer/feeders/Jabber.py b/bin/importer/feeders/Jabber.py index 79d0950f..8c90adfd 100755 --- a/bin/importer/feeders/Jabber.py +++ b/bin/importer/feeders/Jabber.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class JabberFeeder(DefaultFeeder): @@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder): self.item_id = f'{item_id}.gz' return self.item_id - def process_meta(self): + def process_meta(self): # TODO replace me by message """ Process JSON meta field. """ @@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder): # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ############################################## to = str(self.json_data['meta']['jabber:to']) fr = str(self.json_data['meta']['jabber:from']) - date = item_basic.get_item_date(item_id) + + item = Item(self.item_id) + date = item.get_date() user_to = Username(to, 'jabber') user_fr = Username(fr, 'jabber') - user_to.add(date, self.item_id) - user_fr.add(date, self.item_id) + user_to.add(date, item) + user_fr.add(date, item) return None diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 2900a46d..2cc6a127 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -21,7 +21,6 @@ from lib.objects.Chats import Chat from lib.objects import Messages from lib.objects import UsersAccount from lib.objects.Usernames import Username -from lib import item_basic import base64 import io @@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder): # date = datetime.date.today().strftime("%Y/%m/%d") chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id return self.item_id def process_meta(self): @@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder): meta = self.json_data['meta'] mess_id = self.json_data['meta']['id'] if meta.get('reply_to'): - reply_to_id = meta['reply_to'] + reply_to_id = meta['reply_to']['id'] else: reply_to_id = None @@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder): date = datetime.datetime.fromtimestamp(timestamp) date = date.strftime('%Y%m%d') + if self.json_data.get('translation'): + translation = self.json_data['translation'] + else: + translation = None + decoded = base64.standard_b64decode(self.json_data['data']) + content = gunzip_bytes_obj(decoded) + message = Messages.create(self.item_id, content, translation=translation) + if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') - if meta['chat'].get('username'): # SAVE USERNAME - chat_username = meta['chat']['username'] + if meta['chat'].get('username'): + chat_username = Username(meta['chat']['username'], 'telegram') + chat.update_username_timeline(chat_username.get_global_id(), timestamp) # Chat---Message - chat.add(date, self.item_id) # TODO modify to accept file objects - # message meta ????? who is the user if two user ???? - - if self.json_data.get('translation'): - translation = self.json_data['translation'] - else: - translation = None - decoded = base64.standard_b64decode(self.json_data['data']) - content = gunzip_bytes_obj(decoded) - Messages.create(self.item_id, content, translation=translation) - - chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) + chat.add(date) + chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id) else: chat = None @@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder): user_id = meta['sender']['id'] user_account = UsersAccount.UserAccount(user_id, 'telegram') # UserAccount---Message - user_account.add(date, self.item_id) + user_account.add(date, obj=message) # UserAccount---Chat user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) @@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder): if meta['sender'].get('username'): username = Username(meta['sender']['username'], 'telegram') + # TODO timeline or/and correlation ???? user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - # TODO Update user_account<--->username timeline + user_account.update_username_timeline(username.get_global_id(), timestamp) # Username---Message - username.add(date, self.item_id) # TODO #################################################################### + username.add(date) # TODO # correlation message ??? - if chat: - # Chat---Username - chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + # if chat: # TODO Chat---Username correlation ??? + # # Chat---Username + # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # if meta.get('fwd_from'): # if meta['fwd_from'].get('post_author') # user first name # TODO reply threads ???? + # message edit ???? return None diff --git a/bin/importer/feeders/Twitter.py b/bin/importer/feeders/Twitter.py index d5040c65..1c719e73 100755 --- a/bin/importer/feeders/Twitter.py +++ b/bin/importer/feeders/Twitter.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class TwitterFeeder(DefaultFeeder): @@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder): ''' # tweet_id = str(self.json_data['meta']['twitter:tweet_id']) # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################ - - date = item_basic.get_item_date(self.item_id) + item = Item(self.item_id) + date = item.get_date() user = str(self.json_data['meta']['twitter:id']) username = Username(user, 'twitter') - username.add(date, item_id) + username.add(date, item) return None diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 94a06773..f7b13f61 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,20 +41,22 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { - "chat": ["item", "username"], # item ??? + "chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ??? "cookie-name": ["domain"], - "cryptocurrency": ["domain", "item"], - "cve": ["domain", "item"], - "decoded": ["domain", "item"], + "cryptocurrency": ["domain", "item", "message"], + "cve": ["domain", "item", "message"], + "decoded": ["domain", "item", "message"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "hhhash": ["domain"], - "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], - "pgp": ["domain", "item"], + "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? + "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ?? + "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "username": ["chat", "domain", "item"], + "user-account": ["chat", "message"], + "username": ["domain", "item", "message"], # TODO chat-user/account } def get_obj_correl_types(obj_type): diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3e61ed88..6387c76f 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name(): for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): print(domain, date, cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(date, domain) + cookie.add(date, Domain(domain)) def extract_etag_from_har(har): # TODO check response url etags = set() @@ -365,7 +365,7 @@ def _reprocess_all_hars_etag(): for etag_content in extract_etag_from_har(get_har_content(har_id)): print(domain, date, etag_content) etag = Etags.create(etag_content) - etag.add(date, domain) + etag.add(date, Domain(domain)) def extract_hhhash_by_id(har_id, domain, date): return extract_hhhash(get_har_content(har_id), domain, date) @@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date): # ----- obj = HHHashs.create(hhhash_header, hhhash) - obj.add(date, domain) + obj.add(date, Domain(domain)) hhhashs.add(hhhash) urls.add(url) diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index a3d1721c..bb27413d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.data_retention_engine import update_obj_date from lib.objects import ail_objects -from lib import item_basic +from lib.timeline_engine import Timeline from lib.correlations_engine import get_correlation_by_correl_type @@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? users.add(account[1:]) return users + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') + + def get_username(self): + return self._get_timeline_username().get_last_obj_id() + + def get_usernames(self): + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) + # def get_last_message_id(self): # @@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): obj = ail_objects.get_obj_from_global_id(obj_global_id) - mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) + mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) if mess_dict.get('parent') and parent: mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) - mess_dict['username'] = {} - user = obj.get_correlation('username').get('username') - if user: - subtype, user = user.pop().split(':', 1) - mess_dict['username']['type'] = 'telegram' - mess_dict['username']['subtype'] = subtype - mess_dict['username']['id'] = user + if mess_dict.get('user-account'): + user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account']) + mess_dict['user-account'] = {} + mess_dict['user-account']['type'] = user_account.get_type() + mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) + mess_dict['user-account']['id'] = user_account.get_id() + username = user_account.get_username() + if username: + username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False) + mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? else: - mess_dict['username']['id'] = 'UNKNOWN' + mess_dict['user-account']['id'] = 'UNKNOWN' if not mess_datetime: obj_mess_id = self._get_message_timestamp(obj_global_id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 302f0d0a..b724f854 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -27,7 +27,7 @@ from flask import url_for config_loader = ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") r_object = config_loader.get_db_conn("Kvrocks_Objects") -r_content = config_loader.get_db_conn("Kvrocks_Content") +# r_content = config_loader.get_db_conn("Kvrocks_Content") baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -61,7 +61,7 @@ class Message(AbstractObject): """ Returns source/feeder name """ - l_source = self.id.split('/')[:-4] + l_source = self.id.split('/')[:-2] return os.path.join(*l_source) def get_basename(self): @@ -79,7 +79,7 @@ class Message(AbstractObject): def get_date(self): timestamp = self.get_timestamp() - return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') def get_timestamp(self): dirs = self.id.split('/') @@ -92,11 +92,16 @@ class Message(AbstractObject): return message_id def get_chat_id(self): # TODO optimize -> use me to tag Chat - chat_id = self.get_basename().rsplit('_', 1)[0] + chat_id = self.get_basename().rsplit('_', 1)[0] # if chat_id.endswith('.gz'): # chat_id = chat_id[:-3] return chat_id + def get_user_account(self): + user_account = self.get_correlation('user-account') + if user_account.get('user-account'): + return f'user-account:{user_account["user-account"].pop()}' + # Update value on import # reply to -> parent ? # reply/comment - > children ? @@ -139,7 +144,7 @@ class Message(AbstractObject): return url def get_svg_icon(self): - return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} + return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5} def get_misp_object(self): # TODO obj = MISPObject('instant-message', standalone=True) @@ -181,6 +186,8 @@ class Message(AbstractObject): meta['investigations'] = self.get_investigations() if 'link' in options: meta['link'] = self.get_link(flask_context=True) + if 'user-account' in options: + meta['user-account'] = self.get_user_account() # meta['encoding'] = None return meta @@ -238,7 +245,7 @@ class Message(AbstractObject): def create(self, content, translation, tags): self._set_field('content', content) - r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) + # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) if translation: self._set_translation(translation) for tag in tags: diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f4f71d05..5bc94a9c 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -3,7 +3,7 @@ import os import sys -import re +# import re from flask import url_for from pymisp import MISPObject @@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib import ail_core from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.timeline_engine import Timeline config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject): def set_phone(self, phone): return self._set_field('phone', phone) - # TODO REWRITE ADD FUNCTION + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') def get_username(self): - return '' + return self._get_timeline_username().get_last_obj_id() def get_usernames(self): - usernames = [] - # TODO TIMELINE - return usernames + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) def get_meta(self, options=set()): meta = self._get_meta(options=options) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 59a7e968..a3f25216 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -65,12 +65,14 @@ class AbstractObject(ABC): def get_global_id(self): return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}' - def get_default_meta(self, tags=False): + def get_default_meta(self, tags=False, link=False): dict_meta = {'id': self.get_id(), 'type': self.get_type(), 'subtype': self.get_subtype(r_str=True)} if tags: dict_meta['tags'] = self.get_tags() + if link: + dict_meta['link'] = self.get_link() return dict_meta def _get_field(self, field): diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 82bb85f6..007f716b 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC): # # - def add(self, date, item_id): + def add(self, date, obj=None): self.update_daterange(date) update_obj_date(date, self.type, self.subtype) # daily @@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC): ####################################################################### ####################################################################### - # Correlations - self.add_correlation('item', '', item_id) - # domain - if is_crawled(item_id): - domain = get_item_domain(item_id) - self.add_correlation('domain', '', domain) + if obj: + # Correlations + self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id()) + if obj.type == 'item': # TODO same for message->chat ??? + item_id = obj.get_id() + # domain + if is_crawled(item_id): + domain = get_item_domain(item_id) + self.add_correlation('domain', '', domain) # TODO:ADD objects + Stats def create(self, first_seen, last_seen): self.set_first_seen(first_seen) self.set_last_seen(last_seen) - def _delete(self): pass diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index cd2f7225..3a19060a 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -23,9 +23,11 @@ from lib.objects import Etags from lib.objects.Favicons import Favicon from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects +from lib.objects.Messages import Message from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles +from lib.objects.UsersAccount import UserAccount from lib.objects import Usernames config_loader = ConfigLoader() @@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id): return Favicon(id) elif obj_type == 'hhhash': return HHHashs.HHHash(id) + elif obj_type == 'message': + return Message(id) elif obj_type == 'screenshot': return Screenshot(id) elif obj_type == 'cryptocurrency': @@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id): return Pgps.Pgp(id, subtype) elif obj_type == 'title': return Titles.Title(id) + elif obj_type == 'user-account': + return UserAccount(id, subtype) elif obj_type == 'username': return Usernames.Username(id, subtype) diff --git a/bin/lib/timeline_engine.py b/bin/lib/timeline_engine.py index 405e7a50..58c222f6 100755 --- a/bin/lib/timeline_engine.py +++ b/bin/lib/timeline_engine.py @@ -46,112 +46,167 @@ config_loader = None # return [] # return correl_types -# TODO rename all function + add missing parameters +class Timeline: -def get_bloc_obj_global_id(bloc): - return r_meta.hget('hset:key', bloc) + def __init__(self, global_id, name): + self.id = global_id + self.name = name -def set_bloc_obj_global_id(bloc, global_id): - return r_meta.hset('hset:key', bloc, global_id) + def _get_block_obj_global_id(self, block): + return r_meta.hget(f'block:{self.id}:{self.name}', block) -def get_bloc_timestamp(bloc, position): - return r_meta.zscore('key', f'{position}:{bloc}') + def _set_block_obj_global_id(self, block, global_id): + return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id) -def add_bloc(global_id, timestamp, end=None): - if end: - timestamp_end = end - else: - timestamp_end = timestamp - new_bloc = str(uuid4()) - r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) - set_bloc_obj_global_id(new_bloc, global_id) - return new_bloc + def _get_block_timestamp(self, block, position): + return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}') -def _update_bloc(bloc, position, timestamp): - r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) + def _get_nearest_bloc_inf(self, timestamp): + inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True) + if inf: + inf, score = inf[0] + if inf.startswith('end'): + inf_key = f'start:{inf[4:]}' + inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key) + if inf_score == score: + inf = inf_key + return inf + else: + return None -# score = timestamp -def get_nearest_bloc_inf(timestamp): - return r_meta.zrevrangebyscore('key', timestamp, 0, num=1) + def _get_nearest_bloc_sup(self, timestamp): + sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True) + if sup: + sup, score = sup[0] + if sup.startswith('start'): + sup_key = f'end:{sup[6:]}' + sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key) + if score == sup_score: + sup = sup_key + return sup + else: + return None -def get_nearest_bloc_sup(timestamp): - return r_meta.zrangebyscore('key', timestamp, 0, num=1) + def get_first_obj_id(self): + first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0) + if first: # start:block + first = first[0] + if first.startswith('start:'): + first = first[6:] + else: + first = first[4:] + return self._get_block_obj_global_id(first) -####################################################################################### + def get_last_obj_id(self): + last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0) + if last: # end:block + last = last[0] + if last.startswith('end:'): + last = last[4:] + else: + last = last[6:] + return self._get_block_obj_global_id(last) -def add_timestamp(timestamp, obj_global_id): - inf = get_nearest_bloc_inf(timestamp) - sup = get_nearest_bloc_sup(timestamp) - if not inf and not sup: - # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) + def get_objs_ids(self): + objs = set() + for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1): + if block: + if block.startswith('start:'): + objs.add(self._get_block_obj_global_id(block[6:])) + return objs + + # def get_objs_ids(self): + # objs = {} + # last_obj_id = None + # for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True): + # if block: + # if block.startswith('start:'): + # last_obj_id = self._get_block_obj_global_id(block[6:]) + # objs[last_obj_id] = {'first_seen': timestamp} + # else: + # objs[last_obj_id]['last_seen'] = timestamp + # return objs + + def _update_bloc(self, block, position, timestamp): + r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp}) + + def _add_bloc(self, obj_global_id, timestamp, end=None): + if end: + timestamp_end = end + else: + timestamp_end = timestamp + new_bloc = str(uuid4()) + r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) + self._set_block_obj_global_id(new_bloc, obj_global_id) return new_bloc - # timestamp < first_seen - elif not inf: - sup_pos, sup_id = inf.split(':') - sup_obj = get_bloc_obj_global_id(sup_pos) - if sup_obj == obj_global_id: - _update_bloc(sup_id, 'start', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) + + def add_timestamp(self, timestamp, obj_global_id): + inf = self._get_nearest_bloc_inf(timestamp) + sup = self._get_nearest_bloc_sup(timestamp) + if not inf and not sup: + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - - # timestamp > first_seen - elif not sup: - inf_pos, inf_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - if inf_obj == obj_global_id: - _update_bloc(inf_id, 'end', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc - - else: - inf_pos, inf_id = inf.split(':') - sup_pos, sup_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - - if inf_id == sup_id: - # reduce bloc + create two new bloc - if obj_global_id != inf_obj: - # get end timestamp - sup_timestamp = get_bloc_timestamp(sup_id, 'end') - # reduce original bloc - _update_bloc(inf_id, 'end', timestamp - 1) - # Insert new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - # Recreate end of the first bloc by a new bloc - add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + # timestamp < first_seen + elif not inf: + sup_pos, sup_id = sup.split(':') + sup_obj = self._get_block_obj_global_id(sup_id) + if sup_obj == obj_global_id: + self._update_bloc(sup_id, 'start', timestamp) + # create new bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - # timestamp in existing bloc - else: - return inf_id - - # different blocs: expend sup/inf bloc or create a new bloc if - elif inf_pos == 'end' and sup_pos == 'start': - # Extend inf bloc - if obj_global_id == inf_obj: - _update_bloc(inf_id, 'end', timestamp) - return inf_id - - sup_obj = get_bloc_obj_global_id(sup_pos) - # Extend sup bloc - if obj_global_id == sup_obj: - _update_bloc(sup_id, 'start', timestamp) - return sup_id - + # timestamp > first_seen + elif not sup: + inf_pos, inf_id = inf.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_obj == obj_global_id: + self._update_bloc(inf_id, 'end', timestamp) # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc - # inf_pos == 'start' and sup_pos == 'end' - # else raise error ??? + else: + inf_pos, inf_id = inf.split(':') + sup_pos, sup_id = sup.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_id == sup_id: + # reduce bloc + create two new bloc + if obj_global_id != inf_obj: + # get end timestamp + sup_timestamp = self._get_block_timestamp(sup_id, 'end') + # reduce original bloc + self._update_bloc(inf_id, 'end', timestamp - 1) + # Insert new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + # Recreate end of the first bloc by a new bloc + self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + return new_bloc + # timestamp in existing bloc + else: + return inf_id + # different blocs: expend sup/inf bloc or create a new bloc if + elif inf_pos == 'end' and sup_pos == 'start': + # Extend inf bloc + if obj_global_id == inf_obj: + self._update_bloc(inf_id, 'end', timestamp) + return inf_id + sup_obj = self._get_block_obj_global_id(sup_id) + # Extend sup bloc + if obj_global_id == sup_obj: + self._update_bloc(sup_id, 'start', timestamp) + return sup_id + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc + # inf_pos == 'start' and sup_pos == 'end' + # else raise error ??? diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 6197f8a1..318cdd88 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC): if crypto.is_valid_address(): # print(address) is_valid_address = True - crypto.add(date, item_id) + crypto.add(date, item) # Check private key if is_valid_address: diff --git a/bin/modules/PgpDump.py b/bin/modules/PgpDump.py index 0647e897..53c89a19 100755 --- a/bin/modules/PgpDump.py +++ b/bin/modules/PgpDump.py @@ -210,18 +210,18 @@ class PgpDump(AbstractModule): date = item.get_date() for key in self.keys: pgp = Pgps.Pgp(key, 'key') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' key: {key}') for name in self.names: pgp = Pgps.Pgp(name, 'name') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' name: {name}') self.tracker_term.compute(name, obj_type='pgp', subtype='name') self.tracker_regex.compute(name, obj_type='pgp', subtype='name') self.tracker_yara.compute(name, obj_type='pgp', subtype='name') for mail in self.mails: pgp = Pgps.Pgp(mail, 'mail') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' mail: {mail}') self.tracker_term.compute(mail, obj_type='pgp', subtype='mail') self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail') diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 31d90878..42feaa09 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -58,7 +58,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: @@ -73,7 +73,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html index b89a447a..aa0bdf35 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -120,14 +120,26 @@
- {{ mess['username']['id'] }} + {{ mess['user-account']['id'] }}
{{ mess['hour'] }}
-
{{ mess['username']['id'] }}
+
+ {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
{% if mess['reply_to'] %}
-
{{ mess['reply_to']['username']['id'] }}
+
+ {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} + 
{{ mess['reply_to']['content'] }}
{% for tag in mess['reply_to']['tags'] %} {{ tag }}