chg: [chat + user-account] correlations + usernames timeline

This commit is contained in:
Terrtia 2023-08-28 16:29:38 +02:00
parent 843b2d3134
commit b32f110285
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
17 changed files with 279 additions and 173 deletions

View file

@ -282,7 +282,7 @@ class Crawler(AbstractModule):
title_content = crawlers.extract_title_from_html(entries['html']) title_content = crawlers.extract_title_from_html(entries['html'])
if title_content: if title_content:
title = Titles.create_title(title_content) title = Titles.create_title(title_content)
title.add(item.get_date(), item_id) title.add(item.get_date(), item)
# SCREENSHOT # SCREENSHOT
if self.screenshot: if self.screenshot:
@ -306,11 +306,11 @@ class Crawler(AbstractModule):
for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']): for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']):
print(cookie_name) print(cookie_name)
cookie = CookiesNames.create(cookie_name) cookie = CookiesNames.create(cookie_name)
cookie.add(self.date.replace('/', ''), self.domain.id) cookie.add(self.date.replace('/', ''), self.domain)
for etag_content in crawlers.extract_etag_from_har(entries['har']): for etag_content in crawlers.extract_etag_from_har(entries['har']):
print(etag_content) print(etag_content)
etag = Etags.create(etag_content) etag = Etags.create(etag_content)
etag.add(self.date.replace('/', ''), self.domain.id) etag.add(self.date.replace('/', ''), self.domain)
crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', '')) crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', ''))
# Next Children # Next Children

View file

@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from importer.feeders.Default import DefaultFeeder from importer.feeders.Default import DefaultFeeder
from lib.objects.Usernames import Username from lib.objects.Usernames import Username
from lib import item_basic from lib.objects.Items import Item
class JabberFeeder(DefaultFeeder): class JabberFeeder(DefaultFeeder):
@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder):
self.item_id = f'{item_id}.gz' self.item_id = f'{item_id}.gz'
return self.item_id return self.item_id
def process_meta(self): def process_meta(self): # TODO replace me by message
""" """
Process JSON meta field. Process JSON meta field.
""" """
@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder):
# item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ############################################## # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ##############################################
to = str(self.json_data['meta']['jabber:to']) to = str(self.json_data['meta']['jabber:to'])
fr = str(self.json_data['meta']['jabber:from']) fr = str(self.json_data['meta']['jabber:from'])
date = item_basic.get_item_date(item_id)
item = Item(self.item_id)
date = item.get_date()
user_to = Username(to, 'jabber') user_to = Username(to, 'jabber')
user_fr = Username(fr, 'jabber') user_fr = Username(fr, 'jabber')
user_to.add(date, self.item_id) user_to.add(date, item)
user_fr.add(date, self.item_id) user_fr.add(date, item)
return None return None

View file

@ -21,7 +21,6 @@ from lib.objects.Chats import Chat
from lib.objects import Messages from lib.objects import Messages
from lib.objects import UsersAccount from lib.objects import UsersAccount
from lib.objects.Usernames import Username from lib.objects.Usernames import Username
from lib import item_basic
import base64 import base64
import io import io
@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder):
# date = datetime.date.today().strftime("%Y/%m/%d") # date = datetime.date.today().strftime("%Y/%m/%d")
chat_id = str(self.json_data['meta']['chat']['id']) chat_id = str(self.json_data['meta']['chat']['id'])
message_id = str(self.json_data['meta']['id']) message_id = str(self.json_data['meta']['id'])
self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id
return self.item_id return self.item_id
def process_meta(self): def process_meta(self):
@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder):
meta = self.json_data['meta'] meta = self.json_data['meta']
mess_id = self.json_data['meta']['id'] mess_id = self.json_data['meta']['id']
if meta.get('reply_to'): if meta.get('reply_to'):
reply_to_id = meta['reply_to'] reply_to_id = meta['reply_to']['id']
else: else:
reply_to_id = None reply_to_id = None
@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder):
date = datetime.datetime.fromtimestamp(timestamp) date = datetime.datetime.fromtimestamp(timestamp)
date = date.strftime('%Y%m%d') date = date.strftime('%Y%m%d')
if meta.get('chat'):
chat = Chat(meta['chat']['id'], 'telegram')
if meta['chat'].get('username'): # SAVE USERNAME
chat_username = meta['chat']['username']
# Chat---Message
chat.add(date, self.item_id) # TODO modify to accept file objects
# message meta ????? who is the user if two user ????
if self.json_data.get('translation'): if self.json_data.get('translation'):
translation = self.json_data['translation'] translation = self.json_data['translation']
else: else:
translation = None translation = None
decoded = base64.standard_b64decode(self.json_data['data']) decoded = base64.standard_b64decode(self.json_data['data'])
content = gunzip_bytes_obj(decoded) content = gunzip_bytes_obj(decoded)
Messages.create(self.item_id, content, translation=translation) message = Messages.create(self.item_id, content, translation=translation)
chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) if meta.get('chat'):
chat = Chat(meta['chat']['id'], 'telegram')
if meta['chat'].get('username'):
chat_username = Username(meta['chat']['username'], 'telegram')
chat.update_username_timeline(chat_username.get_global_id(), timestamp)
# Chat---Message
chat.add(date)
chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
else: else:
chat = None chat = None
@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder):
user_id = meta['sender']['id'] user_id = meta['sender']['id']
user_account = UsersAccount.UserAccount(user_id, 'telegram') user_account = UsersAccount.UserAccount(user_id, 'telegram')
# UserAccount---Message # UserAccount---Message
user_account.add(date, self.item_id) user_account.add(date, obj=message)
# UserAccount---Chat # UserAccount---Chat
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder):
if meta['sender'].get('username'): if meta['sender'].get('username'):
username = Username(meta['sender']['username'], 'telegram') username = Username(meta['sender']['username'], 'telegram')
# TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# TODO Update user_account<--->username timeline user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message # Username---Message
username.add(date, self.item_id) # TODO #################################################################### username.add(date) # TODO # correlation message ???
if chat: # if chat: # TODO Chat---Username correlation ???
# Chat---Username # # Chat---Username
chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# if meta.get('fwd_from'): # if meta.get('fwd_from'):
# if meta['fwd_from'].get('post_author') # user first name # if meta['fwd_from'].get('post_author') # user first name
# TODO reply threads ???? # TODO reply threads ????
# message edit ????
return None return None

View file

@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from importer.feeders.Default import DefaultFeeder from importer.feeders.Default import DefaultFeeder
from lib.objects.Usernames import Username from lib.objects.Usernames import Username
from lib import item_basic from lib.objects.Items import Item
class TwitterFeeder(DefaultFeeder): class TwitterFeeder(DefaultFeeder):
@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder):
''' '''
# tweet_id = str(self.json_data['meta']['twitter:tweet_id']) # tweet_id = str(self.json_data['meta']['twitter:tweet_id'])
# item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################ # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################
item = Item(self.item_id)
date = item_basic.get_item_date(self.item_id) date = item.get_date()
user = str(self.json_data['meta']['twitter:id']) user = str(self.json_data['meta']['twitter:id'])
username = Username(user, 'twitter') username = Username(user, 'twitter')
username.add(date, item_id) username.add(date, item)
return None return None

View file

@ -41,20 +41,22 @@ config_loader = None
################################## ##################################
CORRELATION_TYPES_BY_OBJ = { CORRELATION_TYPES_BY_OBJ = {
"chat": ["item", "username"], # item ??? "chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ???
"cookie-name": ["domain"], "cookie-name": ["domain"],
"cryptocurrency": ["domain", "item"], "cryptocurrency": ["domain", "item", "message"],
"cve": ["domain", "item"], "cve": ["domain", "item", "message"],
"decoded": ["domain", "item"], "decoded": ["domain", "item", "message"],
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
"etag": ["domain"], "etag": ["domain"],
"favicon": ["domain", "item"], # TODO Decoded "favicon": ["domain", "item"], # TODO Decoded
"hhhash": ["domain"], "hhhash": ["domain"],
"item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
"pgp": ["domain", "item"], "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ??
"pgp": ["domain", "item", "message"],
"screenshot": ["domain", "item"], "screenshot": ["domain", "item"],
"title": ["domain", "item"], "title": ["domain", "item"],
"username": ["chat", "domain", "item"], "user-account": ["chat", "message"],
"username": ["domain", "item", "message"], # TODO chat-user/account
} }
def get_obj_correl_types(obj_type): def get_obj_correl_types(obj_type):

View file

@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name():
for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)):
print(domain, date, cookie_name) print(domain, date, cookie_name)
cookie = CookiesNames.create(cookie_name) cookie = CookiesNames.create(cookie_name)
cookie.add(date, domain) cookie.add(date, Domain(domain))
def extract_etag_from_har(har): # TODO check response url def extract_etag_from_har(har): # TODO check response url
etags = set() etags = set()
@ -365,7 +365,7 @@ def _reprocess_all_hars_etag():
for etag_content in extract_etag_from_har(get_har_content(har_id)): for etag_content in extract_etag_from_har(get_har_content(har_id)):
print(domain, date, etag_content) print(domain, date, etag_content)
etag = Etags.create(etag_content) etag = Etags.create(etag_content)
etag.add(date, domain) etag.add(date, Domain(domain))
def extract_hhhash_by_id(har_id, domain, date): def extract_hhhash_by_id(har_id, domain, date):
return extract_hhhash(get_har_content(har_id), domain, date) return extract_hhhash(get_har_content(har_id), domain, date)
@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date):
# ----- # -----
obj = HHHashs.create(hhhash_header, hhhash) obj = HHHashs.create(hhhash_header, hhhash)
obj.add(date, domain) obj.add(date, Domain(domain))
hhhashs.add(hhhash) hhhashs.add(hhhash)
urls.add(url) urls.add(url)

View file

@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.data_retention_engine import update_obj_date from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects from lib.objects import ail_objects
from lib import item_basic from lib.timeline_engine import Timeline
from lib.correlations_engine import get_correlation_by_correl_type from lib.correlations_engine import get_correlation_by_correl_type
@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
users.add(account[1:]) users.add(account[1:])
return users return users
def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username')
def get_username(self):
return self._get_timeline_username().get_last_obj_id()
def get_usernames(self):
return self._get_timeline_username().get_objs_ids()
def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
# def get_last_message_id(self): # def get_last_message_id(self):
# #
@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
obj = ail_objects.get_obj_from_global_id(obj_global_id) obj = ail_objects.get_obj_from_global_id(obj_global_id)
mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
if mess_dict.get('parent') and parent: if mess_dict.get('parent') and parent:
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
mess_dict['username'] = {} if mess_dict.get('user-account'):
user = obj.get_correlation('username').get('username') user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
if user: mess_dict['user-account'] = {}
subtype, user = user.pop().split(':', 1) mess_dict['user-account']['type'] = user_account.get_type()
mess_dict['username']['type'] = 'telegram' mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
mess_dict['username']['subtype'] = subtype mess_dict['user-account']['id'] = user_account.get_id()
mess_dict['username']['id'] = user username = user_account.get_username()
if username:
username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
else: else:
mess_dict['username']['id'] = 'UNKNOWN' mess_dict['user-account']['id'] = 'UNKNOWN'
if not mess_datetime: if not mess_datetime:
obj_mess_id = self._get_message_timestamp(obj_global_id) obj_mess_id = self._get_message_timestamp(obj_global_id)

View file

@ -27,7 +27,7 @@ from flask import url_for
config_loader = ConfigLoader() config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache") r_cache = config_loader.get_redis_conn("Redis_Cache")
r_object = config_loader.get_db_conn("Kvrocks_Objects") r_object = config_loader.get_db_conn("Kvrocks_Objects")
r_content = config_loader.get_db_conn("Kvrocks_Content") # r_content = config_loader.get_db_conn("Kvrocks_Content")
baseurl = config_loader.get_config_str("Notifications", "ail_domain") baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None config_loader = None
@ -61,7 +61,7 @@ class Message(AbstractObject):
""" """
Returns source/feeder name Returns source/feeder name
""" """
l_source = self.id.split('/')[:-4] l_source = self.id.split('/')[:-2]
return os.path.join(*l_source) return os.path.join(*l_source)
def get_basename(self): def get_basename(self):
@ -79,7 +79,7 @@ class Message(AbstractObject):
def get_date(self): def get_date(self):
timestamp = self.get_timestamp() timestamp = self.get_timestamp()
return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
def get_timestamp(self): def get_timestamp(self):
dirs = self.id.split('/') dirs = self.id.split('/')
@ -97,6 +97,11 @@ class Message(AbstractObject):
# chat_id = chat_id[:-3] # chat_id = chat_id[:-3]
return chat_id return chat_id
def get_user_account(self):
user_account = self.get_correlation('user-account')
if user_account.get('user-account'):
return f'user-account:{user_account["user-account"].pop()}'
# Update value on import # Update value on import
# reply to -> parent ? # reply to -> parent ?
# reply/comment - > children ? # reply/comment - > children ?
@ -139,7 +144,7 @@ class Message(AbstractObject):
return url return url
def get_svg_icon(self): def get_svg_icon(self):
return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5}
def get_misp_object(self): # TODO def get_misp_object(self): # TODO
obj = MISPObject('instant-message', standalone=True) obj = MISPObject('instant-message', standalone=True)
@ -181,6 +186,8 @@ class Message(AbstractObject):
meta['investigations'] = self.get_investigations() meta['investigations'] = self.get_investigations()
if 'link' in options: if 'link' in options:
meta['link'] = self.get_link(flask_context=True) meta['link'] = self.get_link(flask_context=True)
if 'user-account' in options:
meta['user-account'] = self.get_user_account()
# meta['encoding'] = None # meta['encoding'] = None
return meta return meta
@ -238,7 +245,7 @@ class Message(AbstractObject):
def create(self, content, translation, tags): def create(self, content, translation, tags):
self._set_field('content', content) self._set_field('content', content)
r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
if translation: if translation:
self._set_translation(translation) self._set_translation(translation)
for tag in tags: for tag in tags:

View file

@ -3,7 +3,7 @@
import os import os
import sys import sys
import re # import re
from flask import url_for from flask import url_for
from pymisp import MISPObject from pymisp import MISPObject
@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ail_core from lib import ail_core
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.timeline_engine import Timeline
config_loader = ConfigLoader() config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain") baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject):
def set_phone(self, phone): def set_phone(self, phone):
return self._set_field('phone', phone) return self._set_field('phone', phone)
# TODO REWRITE ADD FUNCTION def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username')
def get_username(self): def get_username(self):
return '' return self._get_timeline_username().get_last_obj_id()
def get_usernames(self): def get_usernames(self):
usernames = [] return self._get_timeline_username().get_objs_ids()
# TODO TIMELINE
return usernames def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
def get_meta(self, options=set()): def get_meta(self, options=set()):
meta = self._get_meta(options=options) meta = self._get_meta(options=options)

View file

@ -65,12 +65,14 @@ class AbstractObject(ABC):
def get_global_id(self): def get_global_id(self):
return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}' return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}'
def get_default_meta(self, tags=False): def get_default_meta(self, tags=False, link=False):
dict_meta = {'id': self.get_id(), dict_meta = {'id': self.get_id(),
'type': self.get_type(), 'type': self.get_type(),
'subtype': self.get_subtype(r_str=True)} 'subtype': self.get_subtype(r_str=True)}
if tags: if tags:
dict_meta['tags'] = self.get_tags() dict_meta['tags'] = self.get_tags()
if link:
dict_meta['link'] = self.get_link()
return dict_meta return dict_meta
def _get_field(self, field): def _get_field(self, field):

View file

@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC):
# #
# #
def add(self, date, item_id): def add(self, date, obj=None):
self.update_daterange(date) self.update_daterange(date)
update_obj_date(date, self.type, self.subtype) update_obj_date(date, self.type, self.subtype)
# daily # daily
@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC):
####################################################################### #######################################################################
####################################################################### #######################################################################
if obj:
# Correlations # Correlations
self.add_correlation('item', '', item_id) self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id())
if obj.type == 'item': # TODO same for message->chat ???
item_id = obj.get_id()
# domain # domain
if is_crawled(item_id): if is_crawled(item_id):
domain = get_item_domain(item_id) domain = get_item_domain(item_id)
self.add_correlation('domain', '', domain) self.add_correlation('domain', '', domain)
# TODO:ADD objects + Stats # TODO:ADD objects + Stats
def create(self, first_seen, last_seen): def create(self, first_seen, last_seen):
self.set_first_seen(first_seen) self.set_first_seen(first_seen)
self.set_last_seen(last_seen) self.set_last_seen(last_seen)
def _delete(self): def _delete(self):
pass pass

View file

@ -23,9 +23,11 @@ from lib.objects import Etags
from lib.objects.Favicons import Favicon from lib.objects.Favicons import Favicon
from lib.objects import HHHashs from lib.objects import HHHashs
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects.Messages import Message
from lib.objects import Pgps from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot from lib.objects.Screenshots import Screenshot
from lib.objects import Titles from lib.objects import Titles
from lib.objects.UsersAccount import UserAccount
from lib.objects import Usernames from lib.objects import Usernames
config_loader = ConfigLoader() config_loader = ConfigLoader()
@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id):
return Favicon(id) return Favicon(id)
elif obj_type == 'hhhash': elif obj_type == 'hhhash':
return HHHashs.HHHash(id) return HHHashs.HHHash(id)
elif obj_type == 'message':
return Message(id)
elif obj_type == 'screenshot': elif obj_type == 'screenshot':
return Screenshot(id) return Screenshot(id)
elif obj_type == 'cryptocurrency': elif obj_type == 'cryptocurrency':
@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id):
return Pgps.Pgp(id, subtype) return Pgps.Pgp(id, subtype)
elif obj_type == 'title': elif obj_type == 'title':
return Titles.Title(id) return Titles.Title(id)
elif obj_type == 'user-account':
return UserAccount(id, subtype)
elif obj_type == 'username': elif obj_type == 'username':
return Usernames.Username(id, subtype) return Usernames.Username(id, subtype)

View file

@ -46,84 +46,145 @@ config_loader = None
# return [] # return []
# return correl_types # return correl_types
# TODO rename all function + add missing parameters class Timeline:
def get_bloc_obj_global_id(bloc): def __init__(self, global_id, name):
return r_meta.hget('hset:key', bloc) self.id = global_id
self.name = name
def set_bloc_obj_global_id(bloc, global_id): def _get_block_obj_global_id(self, block):
return r_meta.hset('hset:key', bloc, global_id) return r_meta.hget(f'block:{self.id}:{self.name}', block)
def get_bloc_timestamp(bloc, position): def _set_block_obj_global_id(self, block, global_id):
return r_meta.zscore('key', f'{position}:{bloc}') return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id)
def add_bloc(global_id, timestamp, end=None): def _get_block_timestamp(self, block, position):
return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}')
def _get_nearest_bloc_inf(self, timestamp):
inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True)
if inf:
inf, score = inf[0]
if inf.startswith('end'):
inf_key = f'start:{inf[4:]}'
inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key)
if inf_score == score:
inf = inf_key
return inf
else:
return None
def _get_nearest_bloc_sup(self, timestamp):
sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True)
if sup:
sup, score = sup[0]
if sup.startswith('start'):
sup_key = f'end:{sup[6:]}'
sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key)
if score == sup_score:
sup = sup_key
return sup
else:
return None
def get_first_obj_id(self):
first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0)
if first: # start:block
first = first[0]
if first.startswith('start:'):
first = first[6:]
else:
first = first[4:]
return self._get_block_obj_global_id(first)
def get_last_obj_id(self):
last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0)
if last: # end:block
last = last[0]
if last.startswith('end:'):
last = last[4:]
else:
last = last[6:]
return self._get_block_obj_global_id(last)
def get_objs_ids(self):
objs = set()
for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1):
if block:
if block.startswith('start:'):
objs.add(self._get_block_obj_global_id(block[6:]))
return objs
# def get_objs_ids(self):
# objs = {}
# last_obj_id = None
# for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True):
# if block:
# if block.startswith('start:'):
# last_obj_id = self._get_block_obj_global_id(block[6:])
# objs[last_obj_id] = {'first_seen': timestamp}
# else:
# objs[last_obj_id]['last_seen'] = timestamp
# return objs
def _update_bloc(self, block, position, timestamp):
r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp})
def _add_bloc(self, obj_global_id, timestamp, end=None):
if end: if end:
timestamp_end = end timestamp_end = end
else: else:
timestamp_end = timestamp timestamp_end = timestamp
new_bloc = str(uuid4()) new_bloc = str(uuid4())
r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
set_bloc_obj_global_id(new_bloc, global_id) self._set_block_obj_global_id(new_bloc, obj_global_id)
return new_bloc return new_bloc
def _update_bloc(bloc, position, timestamp): def add_timestamp(self, timestamp, obj_global_id):
r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) inf = self._get_nearest_bloc_inf(timestamp)
sup = self._get_nearest_bloc_sup(timestamp)
# score = timestamp
def get_nearest_bloc_inf(timestamp):
return r_meta.zrevrangebyscore('key', timestamp, 0, num=1)
def get_nearest_bloc_sup(timestamp):
return r_meta.zrangebyscore('key', timestamp, 0, num=1)
#######################################################################################
def add_timestamp(timestamp, obj_global_id):
inf = get_nearest_bloc_inf(timestamp)
sup = get_nearest_bloc_sup(timestamp)
if not inf and not sup: if not inf and not sup:
# create new bloc # create new bloc
new_bloc = add_bloc(obj_global_id, timestamp) new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc return new_bloc
# timestamp < first_seen # timestamp < first_seen
elif not inf: elif not inf:
sup_pos, sup_id = inf.split(':') sup_pos, sup_id = sup.split(':')
sup_obj = get_bloc_obj_global_id(sup_pos) sup_obj = self._get_block_obj_global_id(sup_id)
if sup_obj == obj_global_id: if sup_obj == obj_global_id:
_update_bloc(sup_id, 'start', timestamp) self._update_bloc(sup_id, 'start', timestamp)
# create new bloc # create new bloc
else: else:
new_bloc = add_bloc(obj_global_id, timestamp) new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc return new_bloc
# timestamp > first_seen # timestamp > first_seen
elif not sup: elif not sup:
inf_pos, inf_id = inf.split(':') inf_pos, inf_id = inf.split(':')
inf_obj = get_bloc_obj_global_id(inf_id) inf_obj = self._get_block_obj_global_id(inf_id)
if inf_obj == obj_global_id: if inf_obj == obj_global_id:
_update_bloc(inf_id, 'end', timestamp) self._update_bloc(inf_id, 'end', timestamp)
# create new bloc # create new bloc
else: else:
new_bloc = add_bloc(obj_global_id, timestamp) new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc return new_bloc
else: else:
inf_pos, inf_id = inf.split(':') inf_pos, inf_id = inf.split(':')
sup_pos, sup_id = inf.split(':') sup_pos, sup_id = sup.split(':')
inf_obj = get_bloc_obj_global_id(inf_id) inf_obj = self._get_block_obj_global_id(inf_id)
if inf_id == sup_id: if inf_id == sup_id:
# reduce bloc + create two new bloc # reduce bloc + create two new bloc
if obj_global_id != inf_obj: if obj_global_id != inf_obj:
# get end timestamp # get end timestamp
sup_timestamp = get_bloc_timestamp(sup_id, 'end') sup_timestamp = self._get_block_timestamp(sup_id, 'end')
# reduce original bloc # reduce original bloc
_update_bloc(inf_id, 'end', timestamp - 1) self._update_bloc(inf_id, 'end', timestamp - 1)
# Insert new bloc # Insert new bloc
new_bloc = add_bloc(obj_global_id, timestamp) new_bloc = self._add_bloc(obj_global_id, timestamp)
# Recreate end of the first bloc by a new bloc # Recreate end of the first bloc by a new bloc
add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
return new_bloc return new_bloc
# timestamp in existing bloc # timestamp in existing bloc
@ -134,24 +195,18 @@ def add_timestamp(timestamp, obj_global_id):
elif inf_pos == 'end' and sup_pos == 'start': elif inf_pos == 'end' and sup_pos == 'start':
# Extend inf bloc # Extend inf bloc
if obj_global_id == inf_obj: if obj_global_id == inf_obj:
_update_bloc(inf_id, 'end', timestamp) self._update_bloc(inf_id, 'end', timestamp)
return inf_id return inf_id
sup_obj = get_bloc_obj_global_id(sup_pos) sup_obj = self._get_block_obj_global_id(sup_id)
# Extend sup bloc # Extend sup bloc
if obj_global_id == sup_obj: if obj_global_id == sup_obj:
_update_bloc(sup_id, 'start', timestamp) self._update_bloc(sup_id, 'start', timestamp)
return sup_id return sup_id
# create new bloc # create new bloc
new_bloc = add_bloc(obj_global_id, timestamp) new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc return new_bloc
# inf_pos == 'start' and sup_pos == 'end' # inf_pos == 'start' and sup_pos == 'end'
# else raise error ??? # else raise error ???

View file

@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC):
if crypto.is_valid_address(): if crypto.is_valid_address():
# print(address) # print(address)
is_valid_address = True is_valid_address = True
crypto.add(date, item_id) crypto.add(date, item)
# Check private key # Check private key
if is_valid_address: if is_valid_address:

View file

@ -210,18 +210,18 @@ class PgpDump(AbstractModule):
date = item.get_date() date = item.get_date()
for key in self.keys: for key in self.keys:
pgp = Pgps.Pgp(key, 'key') pgp = Pgps.Pgp(key, 'key')
pgp.add(date, self.item_id) pgp.add(date, item)
print(f' key: {key}') print(f' key: {key}')
for name in self.names: for name in self.names:
pgp = Pgps.Pgp(name, 'name') pgp = Pgps.Pgp(name, 'name')
pgp.add(date, self.item_id) pgp.add(date, item)
print(f' name: {name}') print(f' name: {name}')
self.tracker_term.compute(name, obj_type='pgp', subtype='name') self.tracker_term.compute(name, obj_type='pgp', subtype='name')
self.tracker_regex.compute(name, obj_type='pgp', subtype='name') self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
self.tracker_yara.compute(name, obj_type='pgp', subtype='name') self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
for mail in self.mails: for mail in self.mails:
pgp = Pgps.Pgp(mail, 'mail') pgp = Pgps.Pgp(mail, 'mail')
pgp.add(date, self.item_id) pgp.add(date, item)
print(f' mail: {mail}') print(f' mail: {mail}')
self.tracker_term.compute(mail, obj_type='pgp', subtype='mail') self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail') self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')

View file

@ -58,7 +58,7 @@ class Telegram(AbstractModule):
user_id = dict_url.get('username') user_id = dict_url.get('username')
if user_id: if user_id:
username = Username(user_id, 'telegram') username = Username(user_id, 'telegram')
username.add(item_date, item.id) username.add(item_date, item)
print(f'username: {user_id}') print(f'username: {user_id}')
invite_hash = dict_url.get('invite_hash') invite_hash = dict_url.get('invite_hash')
if invite_hash: if invite_hash:
@ -73,7 +73,7 @@ class Telegram(AbstractModule):
user_id = dict_url.get('username') user_id = dict_url.get('username')
if user_id: if user_id:
username = Username(user_id, 'telegram') username = Username(user_id, 'telegram')
username.add(item_date, item.id) username.add(item_date, item)
print(f'username: {user_id}') print(f'username: {user_id}')
invite_hash = dict_url.get('invite_hash') invite_hash = dict_url.get('invite_hash')
if invite_hash: if invite_hash:

View file

@ -120,14 +120,26 @@
<div class="chat-message-left pb-0"> <div class="chat-message-left pb-0">
<div> <div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['username']['id'] }}" width="40" height="40"> <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div> <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
</div> </div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto"> <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
<div class="font-weight-bold mb-1">{{ mess['username']['id'] }}</div> <div class="font-weight-bold mb-1">
{% if mess['user-account']['username'] %}
{{ mess['user-account']['username']['id'] }}
{% else %}
{{ mess['user-account']['id'] }}
{% endif %}
</div>
{% if mess['reply_to'] %} {% if mess['reply_to'] %}
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto"> <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<div class="font-weight-bold mb-1">{{ mess['reply_to']['username']['id'] }}</div> <div class="font-weight-bold mb-1">
{% if mess['reply_to']['user-account']['username'] %}
{{ mess['reply_to']['user-account']['username']['id'] }}
{% else %}
{{ mess['reply_to']['user-account']['id'] }}
{% endif %}
</div>
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre> <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
{% for tag in mess['reply_to']['tags'] %} {% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span> <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>