mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-09-19 23:48:23 +00:00
chg: [chat + user-account] correlations + usernames timeline
This commit is contained in:
Terrtia
parent
843b2d3134
commit
b32f110285
17 changed files with 279 additions and 173 deletions
|
|
@ -282,7 +282,7 @@ class Crawler(AbstractModule):
|
|||
title_content = crawlers.extract_title_from_html(entries['html'])
|
||||
if title_content:
|
||||
title = Titles.create_title(title_content)
|
||||
title.add(item.get_date(), item_id)
|
||||
title.add(item.get_date(), item)
|
||||
|
||||
# SCREENSHOT
|
||||
if self.screenshot:
|
||||
|
|
@ -306,11 +306,11 @@ class Crawler(AbstractModule):
|
|||
for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']):
|
||||
print(cookie_name)
|
||||
cookie = CookiesNames.create(cookie_name)
|
||||
cookie.add(self.date.replace('/', ''), self.domain.id)
|
||||
cookie.add(self.date.replace('/', ''), self.domain)
|
||||
for etag_content in crawlers.extract_etag_from_har(entries['har']):
|
||||
print(etag_content)
|
||||
etag = Etags.create(etag_content)
|
||||
etag.add(self.date.replace('/', ''), self.domain.id)
|
||||
etag.add(self.date.replace('/', ''), self.domain)
|
||||
crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', ''))
|
||||
|
||||
# Next Children
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from importer.feeders.Default import DefaultFeeder
|
||||
from lib.objects.Usernames import Username
|
||||
from lib import item_basic
|
||||
from lib.objects.Items import Item
|
||||
|
||||
|
||||
class JabberFeeder(DefaultFeeder):
|
||||
|
|
@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder):
|
|||
self.item_id = f'{item_id}.gz'
|
||||
return self.item_id
|
||||
|
||||
def process_meta(self):
|
||||
def process_meta(self): # TODO replace me by message
|
||||
"""
|
||||
Process JSON meta field.
|
||||
"""
|
||||
|
|
@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder):
|
|||
# item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ##############################################
|
||||
to = str(self.json_data['meta']['jabber:to'])
|
||||
fr = str(self.json_data['meta']['jabber:from'])
|
||||
date = item_basic.get_item_date(item_id)
|
||||
|
||||
item = Item(self.item_id)
|
||||
date = item.get_date()
|
||||
|
||||
user_to = Username(to, 'jabber')
|
||||
user_fr = Username(fr, 'jabber')
|
||||
user_to.add(date, self.item_id)
|
||||
user_fr.add(date, self.item_id)
|
||||
user_to.add(date, item)
|
||||
user_fr.add(date, item)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ from lib.objects.Chats import Chat
|
|||
from lib.objects import Messages
|
||||
from lib.objects import UsersAccount
|
||||
from lib.objects.Usernames import Username
|
||||
from lib import item_basic
|
||||
|
||||
import base64
|
||||
import io
|
||||
|
|
@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder):
|
|||
# date = datetime.date.today().strftime("%Y/%m/%d")
|
||||
chat_id = str(self.json_data['meta']['chat']['id'])
|
||||
message_id = str(self.json_data['meta']['id'])
|
||||
self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
|
||||
self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id
|
||||
return self.item_id
|
||||
|
||||
def process_meta(self):
|
||||
|
|
@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder):
|
|||
meta = self.json_data['meta']
|
||||
mess_id = self.json_data['meta']['id']
|
||||
if meta.get('reply_to'):
|
||||
reply_to_id = meta['reply_to']
|
||||
reply_to_id = meta['reply_to']['id']
|
||||
else:
|
||||
reply_to_id = None
|
||||
|
||||
|
|
@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder):
|
|||
date = datetime.datetime.fromtimestamp(timestamp)
|
||||
date = date.strftime('%Y%m%d')
|
||||
|
||||
if self.json_data.get('translation'):
|
||||
translation = self.json_data['translation']
|
||||
else:
|
||||
translation = None
|
||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||
content = gunzip_bytes_obj(decoded)
|
||||
message = Messages.create(self.item_id, content, translation=translation)
|
||||
|
||||
if meta.get('chat'):
|
||||
chat = Chat(meta['chat']['id'], 'telegram')
|
||||
|
||||
if meta['chat'].get('username'): # SAVE USERNAME
|
||||
chat_username = meta['chat']['username']
|
||||
if meta['chat'].get('username'):
|
||||
chat_username = Username(meta['chat']['username'], 'telegram')
|
||||
chat.update_username_timeline(chat_username.get_global_id(), timestamp)
|
||||
|
||||
# Chat---Message
|
||||
chat.add(date, self.item_id) # TODO modify to accept file objects
|
||||
# message meta ????? who is the user if two user ????
|
||||
|
||||
if self.json_data.get('translation'):
|
||||
translation = self.json_data['translation']
|
||||
else:
|
||||
translation = None
|
||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||
content = gunzip_bytes_obj(decoded)
|
||||
Messages.create(self.item_id, content, translation=translation)
|
||||
|
||||
chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id)
|
||||
chat.add(date)
|
||||
chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
|
||||
else:
|
||||
chat = None
|
||||
|
||||
|
|
@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder):
|
|||
user_id = meta['sender']['id']
|
||||
user_account = UsersAccount.UserAccount(user_id, 'telegram')
|
||||
# UserAccount---Message
|
||||
user_account.add(date, self.item_id)
|
||||
user_account.add(date, obj=message)
|
||||
# UserAccount---Chat
|
||||
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
|
||||
|
||||
|
|
@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder):
|
|||
|
||||
if meta['sender'].get('username'):
|
||||
username = Username(meta['sender']['username'], 'telegram')
|
||||
# TODO timeline or/and correlation ????
|
||||
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
# TODO Update user_account<--->username timeline
|
||||
user_account.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
||||
# Username---Message
|
||||
username.add(date, self.item_id) # TODO ####################################################################
|
||||
username.add(date) # TODO # correlation message ???
|
||||
|
||||
if chat:
|
||||
# Chat---Username
|
||||
chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
# if chat: # TODO Chat---Username correlation ???
|
||||
# # Chat---Username
|
||||
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
|
||||
# if meta.get('fwd_from'):
|
||||
# if meta['fwd_from'].get('post_author') # user first name
|
||||
|
||||
# TODO reply threads ????
|
||||
# message edit ????
|
||||
|
||||
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from importer.feeders.Default import DefaultFeeder
|
||||
from lib.objects.Usernames import Username
|
||||
from lib import item_basic
|
||||
from lib.objects.Items import Item
|
||||
|
||||
class TwitterFeeder(DefaultFeeder):
|
||||
|
||||
|
|
@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder):
|
|||
'''
|
||||
# tweet_id = str(self.json_data['meta']['twitter:tweet_id'])
|
||||
# item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################
|
||||
|
||||
date = item_basic.get_item_date(self.item_id)
|
||||
item = Item(self.item_id)
|
||||
date = item.get_date()
|
||||
user = str(self.json_data['meta']['twitter:id'])
|
||||
username = Username(user, 'twitter')
|
||||
username.add(date, item_id)
|
||||
username.add(date, item)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -41,20 +41,22 @@ config_loader = None
|
|||
##################################
|
||||
|
||||
CORRELATION_TYPES_BY_OBJ = {
|
||||
"chat": ["item", "username"], # item ???
|
||||
"chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ???
|
||||
"cookie-name": ["domain"],
|
||||
"cryptocurrency": ["domain", "item"],
|
||||
"cve": ["domain", "item"],
|
||||
"decoded": ["domain", "item"],
|
||||
"cryptocurrency": ["domain", "item", "message"],
|
||||
"cve": ["domain", "item", "message"],
|
||||
"decoded": ["domain", "item", "message"],
|
||||
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
|
||||
"etag": ["domain"],
|
||||
"favicon": ["domain", "item"], # TODO Decoded
|
||||
"hhhash": ["domain"],
|
||||
"item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
|
||||
"pgp": ["domain", "item"],
|
||||
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
|
||||
"message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ??
|
||||
"pgp": ["domain", "item", "message"],
|
||||
"screenshot": ["domain", "item"],
|
||||
"title": ["domain", "item"],
|
||||
"username": ["chat", "domain", "item"],
|
||||
"user-account": ["chat", "message"],
|
||||
"username": ["domain", "item", "message"], # TODO chat-user/account
|
||||
}
|
||||
|
||||
def get_obj_correl_types(obj_type):
|
||||
|
|
|
|||
|
|
@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name():
|
|||
for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)):
|
||||
print(domain, date, cookie_name)
|
||||
cookie = CookiesNames.create(cookie_name)
|
||||
cookie.add(date, domain)
|
||||
cookie.add(date, Domain(domain))
|
||||
|
||||
def extract_etag_from_har(har): # TODO check response url
|
||||
etags = set()
|
||||
|
|
@ -365,7 +365,7 @@ def _reprocess_all_hars_etag():
|
|||
for etag_content in extract_etag_from_har(get_har_content(har_id)):
|
||||
print(domain, date, etag_content)
|
||||
etag = Etags.create(etag_content)
|
||||
etag.add(date, domain)
|
||||
etag.add(date, Domain(domain))
|
||||
|
||||
def extract_hhhash_by_id(har_id, domain, date):
|
||||
return extract_hhhash(get_har_content(har_id), domain, date)
|
||||
|
|
@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date):
|
|||
|
||||
# -----
|
||||
obj = HHHashs.create(hhhash_header, hhhash)
|
||||
obj.add(date, domain)
|
||||
obj.add(date, Domain(domain))
|
||||
|
||||
hhhashs.add(hhhash)
|
||||
urls.add(url)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
|
|||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib.objects import ail_objects
|
||||
from lib import item_basic
|
||||
from lib.timeline_engine import Timeline
|
||||
|
||||
from lib.correlations_engine import get_correlation_by_correl_type
|
||||
|
||||
|
|
@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
users.add(account[1:])
|
||||
return users
|
||||
|
||||
def _get_timeline_username(self):
|
||||
return Timeline(self.get_global_id(), 'username')
|
||||
|
||||
def get_username(self):
|
||||
return self._get_timeline_username().get_last_obj_id()
|
||||
|
||||
def get_usernames(self):
|
||||
return self._get_timeline_username().get_objs_ids()
|
||||
|
||||
def update_username_timeline(self, username_global_id, timestamp):
|
||||
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
|
||||
|
||||
|
||||
# def get_last_message_id(self):
|
||||
#
|
||||
|
|
@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
|
||||
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
|
||||
obj = ail_objects.get_obj_from_global_id(obj_global_id)
|
||||
mess_dict = obj.get_meta(options={'content', 'link', 'parent'})
|
||||
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
|
||||
if mess_dict.get('parent') and parent:
|
||||
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
|
||||
mess_dict['username'] = {}
|
||||
user = obj.get_correlation('username').get('username')
|
||||
if user:
|
||||
subtype, user = user.pop().split(':', 1)
|
||||
mess_dict['username']['type'] = 'telegram'
|
||||
mess_dict['username']['subtype'] = subtype
|
||||
mess_dict['username']['id'] = user
|
||||
if mess_dict.get('user-account'):
|
||||
user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
|
||||
mess_dict['user-account'] = {}
|
||||
mess_dict['user-account']['type'] = user_account.get_type()
|
||||
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
|
||||
mess_dict['user-account']['id'] = user_account.get_id()
|
||||
username = user_account.get_username()
|
||||
if username:
|
||||
username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
|
||||
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
|
||||
else:
|
||||
mess_dict['username']['id'] = 'UNKNOWN'
|
||||
mess_dict['user-account']['id'] = 'UNKNOWN'
|
||||
|
||||
if not mess_datetime:
|
||||
obj_mess_id = self._get_message_timestamp(obj_global_id)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from flask import url_for
|
|||
config_loader = ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
r_content = config_loader.get_db_conn("Kvrocks_Content")
|
||||
# r_content = config_loader.get_db_conn("Kvrocks_Content")
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
config_loader = None
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ class Message(AbstractObject):
|
|||
"""
|
||||
Returns source/feeder name
|
||||
"""
|
||||
l_source = self.id.split('/')[:-4]
|
||||
l_source = self.id.split('/')[:-2]
|
||||
return os.path.join(*l_source)
|
||||
|
||||
def get_basename(self):
|
||||
|
|
@ -79,7 +79,7 @@ class Message(AbstractObject):
|
|||
|
||||
def get_date(self):
|
||||
timestamp = self.get_timestamp()
|
||||
return datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
|
||||
return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
|
||||
|
||||
def get_timestamp(self):
|
||||
dirs = self.id.split('/')
|
||||
|
|
@ -92,11 +92,16 @@ class Message(AbstractObject):
|
|||
return message_id
|
||||
|
||||
def get_chat_id(self): # TODO optimize -> use me to tag Chat
|
||||
chat_id = self.get_basename().rsplit('_', 1)[0]
|
||||
chat_id = self.get_basename().rsplit('_', 1)[0]
|
||||
# if chat_id.endswith('.gz'):
|
||||
# chat_id = chat_id[:-3]
|
||||
return chat_id
|
||||
|
||||
def get_user_account(self):
|
||||
user_account = self.get_correlation('user-account')
|
||||
if user_account.get('user-account'):
|
||||
return f'user-account:{user_account["user-account"].pop()}'
|
||||
|
||||
# Update value on import
|
||||
# reply to -> parent ?
|
||||
# reply/comment - > children ?
|
||||
|
|
@ -139,7 +144,7 @@ class Message(AbstractObject):
|
|||
return url
|
||||
|
||||
def get_svg_icon(self):
|
||||
return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5}
|
||||
return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5}
|
||||
|
||||
def get_misp_object(self): # TODO
|
||||
obj = MISPObject('instant-message', standalone=True)
|
||||
|
|
@ -181,6 +186,8 @@ class Message(AbstractObject):
|
|||
meta['investigations'] = self.get_investigations()
|
||||
if 'link' in options:
|
||||
meta['link'] = self.get_link(flask_context=True)
|
||||
if 'user-account' in options:
|
||||
meta['user-account'] = self.get_user_account()
|
||||
|
||||
# meta['encoding'] = None
|
||||
return meta
|
||||
|
|
@ -238,7 +245,7 @@ class Message(AbstractObject):
|
|||
|
||||
def create(self, content, translation, tags):
|
||||
self._set_field('content', content)
|
||||
r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
|
||||
# r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
|
||||
if translation:
|
||||
self._set_translation(translation)
|
||||
for tag in tags:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
# import re
|
||||
|
||||
from flask import url_for
|
||||
from pymisp import MISPObject
|
||||
|
|
@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from lib import ail_core
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.timeline_engine import Timeline
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
|
|
@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject):
|
|||
def set_phone(self, phone):
|
||||
return self._set_field('phone', phone)
|
||||
|
||||
# TODO REWRITE ADD FUNCTION
|
||||
def _get_timeline_username(self):
|
||||
return Timeline(self.get_global_id(), 'username')
|
||||
|
||||
def get_username(self):
|
||||
return ''
|
||||
return self._get_timeline_username().get_last_obj_id()
|
||||
|
||||
def get_usernames(self):
|
||||
usernames = []
|
||||
# TODO TIMELINE
|
||||
return usernames
|
||||
return self._get_timeline_username().get_objs_ids()
|
||||
|
||||
def update_username_timeline(self, username_global_id, timestamp):
|
||||
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
|
|
|
|||
|
|
@ -65,12 +65,14 @@ class AbstractObject(ABC):
|
|||
def get_global_id(self):
|
||||
return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}'
|
||||
|
||||
def get_default_meta(self, tags=False):
|
||||
def get_default_meta(self, tags=False, link=False):
|
||||
dict_meta = {'id': self.get_id(),
|
||||
'type': self.get_type(),
|
||||
'subtype': self.get_subtype(r_str=True)}
|
||||
if tags:
|
||||
dict_meta['tags'] = self.get_tags()
|
||||
if link:
|
||||
dict_meta['link'] = self.get_link()
|
||||
return dict_meta
|
||||
|
||||
def _get_field(self, field):
|
||||
|
|
|
|||
|
|
@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
#
|
||||
#
|
||||
|
||||
def add(self, date, item_id):
|
||||
def add(self, date, obj=None):
|
||||
self.update_daterange(date)
|
||||
update_obj_date(date, self.type, self.subtype)
|
||||
# daily
|
||||
|
|
@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
#######################################################################
|
||||
#######################################################################
|
||||
|
||||
# Correlations
|
||||
self.add_correlation('item', '', item_id)
|
||||
# domain
|
||||
if is_crawled(item_id):
|
||||
domain = get_item_domain(item_id)
|
||||
self.add_correlation('domain', '', domain)
|
||||
if obj:
|
||||
# Correlations
|
||||
self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id())
|
||||
|
||||
if obj.type == 'item': # TODO same for message->chat ???
|
||||
item_id = obj.get_id()
|
||||
# domain
|
||||
if is_crawled(item_id):
|
||||
domain = get_item_domain(item_id)
|
||||
self.add_correlation('domain', '', domain)
|
||||
|
||||
# TODO:ADD objects + Stats
|
||||
def create(self, first_seen, last_seen):
|
||||
self.set_first_seen(first_seen)
|
||||
self.set_last_seen(last_seen)
|
||||
|
||||
|
||||
def _delete(self):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -23,9 +23,11 @@ from lib.objects import Etags
|
|||
from lib.objects.Favicons import Favicon
|
||||
from lib.objects import HHHashs
|
||||
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
|
||||
from lib.objects.Messages import Message
|
||||
from lib.objects import Pgps
|
||||
from lib.objects.Screenshots import Screenshot
|
||||
from lib.objects import Titles
|
||||
from lib.objects.UsersAccount import UserAccount
|
||||
from lib.objects import Usernames
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
|
@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id):
|
|||
return Favicon(id)
|
||||
elif obj_type == 'hhhash':
|
||||
return HHHashs.HHHash(id)
|
||||
elif obj_type == 'message':
|
||||
return Message(id)
|
||||
elif obj_type == 'screenshot':
|
||||
return Screenshot(id)
|
||||
elif obj_type == 'cryptocurrency':
|
||||
|
|
@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id):
|
|||
return Pgps.Pgp(id, subtype)
|
||||
elif obj_type == 'title':
|
||||
return Titles.Title(id)
|
||||
elif obj_type == 'user-account':
|
||||
return UserAccount(id, subtype)
|
||||
elif obj_type == 'username':
|
||||
return Usernames.Username(id, subtype)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,112 +46,167 @@ config_loader = None
|
|||
# return []
|
||||
# return correl_types
|
||||
|
||||
# TODO rename all function + add missing parameters
|
||||
class Timeline:
|
||||
|
||||
def get_bloc_obj_global_id(bloc):
|
||||
return r_meta.hget('hset:key', bloc)
|
||||
def __init__(self, global_id, name):
|
||||
self.id = global_id
|
||||
self.name = name
|
||||
|
||||
def set_bloc_obj_global_id(bloc, global_id):
|
||||
return r_meta.hset('hset:key', bloc, global_id)
|
||||
def _get_block_obj_global_id(self, block):
|
||||
return r_meta.hget(f'block:{self.id}:{self.name}', block)
|
||||
|
||||
def get_bloc_timestamp(bloc, position):
|
||||
return r_meta.zscore('key', f'{position}:{bloc}')
|
||||
def _set_block_obj_global_id(self, block, global_id):
|
||||
return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id)
|
||||
|
||||
def add_bloc(global_id, timestamp, end=None):
|
||||
if end:
|
||||
timestamp_end = end
|
||||
else:
|
||||
timestamp_end = timestamp
|
||||
new_bloc = str(uuid4())
|
||||
r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
|
||||
set_bloc_obj_global_id(new_bloc, global_id)
|
||||
return new_bloc
|
||||
def _get_block_timestamp(self, block, position):
|
||||
return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}')
|
||||
|
||||
def _update_bloc(bloc, position, timestamp):
|
||||
r_meta.zadd('key', {f'{position}:{bloc}': timestamp})
|
||||
def _get_nearest_bloc_inf(self, timestamp):
|
||||
inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True)
|
||||
if inf:
|
||||
inf, score = inf[0]
|
||||
if inf.startswith('end'):
|
||||
inf_key = f'start:{inf[4:]}'
|
||||
inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key)
|
||||
if inf_score == score:
|
||||
inf = inf_key
|
||||
return inf
|
||||
else:
|
||||
return None
|
||||
|
||||
# score = timestamp
|
||||
def get_nearest_bloc_inf(timestamp):
|
||||
return r_meta.zrevrangebyscore('key', timestamp, 0, num=1)
|
||||
def _get_nearest_bloc_sup(self, timestamp):
|
||||
sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True)
|
||||
if sup:
|
||||
sup, score = sup[0]
|
||||
if sup.startswith('start'):
|
||||
sup_key = f'end:{sup[6:]}'
|
||||
sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key)
|
||||
if score == sup_score:
|
||||
sup = sup_key
|
||||
return sup
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_nearest_bloc_sup(timestamp):
|
||||
return r_meta.zrangebyscore('key', timestamp, 0, num=1)
|
||||
def get_first_obj_id(self):
|
||||
first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0)
|
||||
if first: # start:block
|
||||
first = first[0]
|
||||
if first.startswith('start:'):
|
||||
first = first[6:]
|
||||
else:
|
||||
first = first[4:]
|
||||
return self._get_block_obj_global_id(first)
|
||||
|
||||
#######################################################################################
|
||||
def get_last_obj_id(self):
|
||||
last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0)
|
||||
if last: # end:block
|
||||
last = last[0]
|
||||
if last.startswith('end:'):
|
||||
last = last[4:]
|
||||
else:
|
||||
last = last[6:]
|
||||
return self._get_block_obj_global_id(last)
|
||||
|
||||
def add_timestamp(timestamp, obj_global_id):
|
||||
inf = get_nearest_bloc_inf(timestamp)
|
||||
sup = get_nearest_bloc_sup(timestamp)
|
||||
if not inf and not sup:
|
||||
# create new bloc
|
||||
new_bloc = add_bloc(obj_global_id, timestamp)
|
||||
def get_objs_ids(self):
|
||||
objs = set()
|
||||
for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1):
|
||||
if block:
|
||||
if block.startswith('start:'):
|
||||
objs.add(self._get_block_obj_global_id(block[6:]))
|
||||
return objs
|
||||
|
||||
# def get_objs_ids(self):
|
||||
# objs = {}
|
||||
# last_obj_id = None
|
||||
# for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True):
|
||||
# if block:
|
||||
# if block.startswith('start:'):
|
||||
# last_obj_id = self._get_block_obj_global_id(block[6:])
|
||||
# objs[last_obj_id] = {'first_seen': timestamp}
|
||||
# else:
|
||||
# objs[last_obj_id]['last_seen'] = timestamp
|
||||
# return objs
|
||||
|
||||
def _update_bloc(self, block, position, timestamp):
|
||||
r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp})
|
||||
|
||||
def _add_bloc(self, obj_global_id, timestamp, end=None):
|
||||
if end:
|
||||
timestamp_end = end
|
||||
else:
|
||||
timestamp_end = timestamp
|
||||
new_bloc = str(uuid4())
|
||||
r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
|
||||
self._set_block_obj_global_id(new_bloc, obj_global_id)
|
||||
return new_bloc
|
||||
# timestamp < first_seen
|
||||
elif not inf:
|
||||
sup_pos, sup_id = inf.split(':')
|
||||
sup_obj = get_bloc_obj_global_id(sup_pos)
|
||||
if sup_obj == obj_global_id:
|
||||
_update_bloc(sup_id, 'start', timestamp)
|
||||
# create new bloc
|
||||
else:
|
||||
new_bloc = add_bloc(obj_global_id, timestamp)
|
||||
|
||||
def add_timestamp(self, timestamp, obj_global_id):
|
||||
inf = self._get_nearest_bloc_inf(timestamp)
|
||||
sup = self._get_nearest_bloc_sup(timestamp)
|
||||
if not inf and not sup:
|
||||
# create new bloc
|
||||
new_bloc = self._add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
|
||||
# timestamp > first_seen
|
||||
elif not sup:
|
||||
inf_pos, inf_id = inf.split(':')
|
||||
inf_obj = get_bloc_obj_global_id(inf_id)
|
||||
if inf_obj == obj_global_id:
|
||||
_update_bloc(inf_id, 'end', timestamp)
|
||||
# create new bloc
|
||||
else:
|
||||
new_bloc = add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
|
||||
else:
|
||||
inf_pos, inf_id = inf.split(':')
|
||||
sup_pos, sup_id = inf.split(':')
|
||||
inf_obj = get_bloc_obj_global_id(inf_id)
|
||||
|
||||
if inf_id == sup_id:
|
||||
# reduce bloc + create two new bloc
|
||||
if obj_global_id != inf_obj:
|
||||
# get end timestamp
|
||||
sup_timestamp = get_bloc_timestamp(sup_id, 'end')
|
||||
# reduce original bloc
|
||||
_update_bloc(inf_id, 'end', timestamp - 1)
|
||||
# Insert new bloc
|
||||
new_bloc = add_bloc(obj_global_id, timestamp)
|
||||
# Recreate end of the first bloc by a new bloc
|
||||
add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
|
||||
# timestamp < first_seen
|
||||
elif not inf:
|
||||
sup_pos, sup_id = sup.split(':')
|
||||
sup_obj = self._get_block_obj_global_id(sup_id)
|
||||
if sup_obj == obj_global_id:
|
||||
self._update_bloc(sup_id, 'start', timestamp)
|
||||
# create new bloc
|
||||
else:
|
||||
new_bloc = self._add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
|
||||
# timestamp in existing bloc
|
||||
else:
|
||||
return inf_id
|
||||
|
||||
# different blocs: expend sup/inf bloc or create a new bloc if
|
||||
elif inf_pos == 'end' and sup_pos == 'start':
|
||||
# Extend inf bloc
|
||||
if obj_global_id == inf_obj:
|
||||
_update_bloc(inf_id, 'end', timestamp)
|
||||
return inf_id
|
||||
|
||||
sup_obj = get_bloc_obj_global_id(sup_pos)
|
||||
# Extend sup bloc
|
||||
if obj_global_id == sup_obj:
|
||||
_update_bloc(sup_id, 'start', timestamp)
|
||||
return sup_id
|
||||
|
||||
# timestamp > first_seen
|
||||
elif not sup:
|
||||
inf_pos, inf_id = inf.split(':')
|
||||
inf_obj = self._get_block_obj_global_id(inf_id)
|
||||
if inf_obj == obj_global_id:
|
||||
self._update_bloc(inf_id, 'end', timestamp)
|
||||
# create new bloc
|
||||
new_bloc = add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
else:
|
||||
new_bloc = self._add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
|
||||
# inf_pos == 'start' and sup_pos == 'end'
|
||||
# else raise error ???
|
||||
else:
|
||||
inf_pos, inf_id = inf.split(':')
|
||||
sup_pos, sup_id = sup.split(':')
|
||||
inf_obj = self._get_block_obj_global_id(inf_id)
|
||||
|
||||
if inf_id == sup_id:
|
||||
# reduce bloc + create two new bloc
|
||||
if obj_global_id != inf_obj:
|
||||
# get end timestamp
|
||||
sup_timestamp = self._get_block_timestamp(sup_id, 'end')
|
||||
# reduce original bloc
|
||||
self._update_bloc(inf_id, 'end', timestamp - 1)
|
||||
# Insert new bloc
|
||||
new_bloc = self._add_bloc(obj_global_id, timestamp)
|
||||
# Recreate end of the first bloc by a new bloc
|
||||
self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
|
||||
return new_bloc
|
||||
|
||||
# timestamp in existing bloc
|
||||
else:
|
||||
return inf_id
|
||||
|
||||
# different blocs: expend sup/inf bloc or create a new bloc if
|
||||
elif inf_pos == 'end' and sup_pos == 'start':
|
||||
# Extend inf bloc
|
||||
if obj_global_id == inf_obj:
|
||||
self._update_bloc(inf_id, 'end', timestamp)
|
||||
return inf_id
|
||||
|
||||
sup_obj = self._get_block_obj_global_id(sup_id)
|
||||
# Extend sup bloc
|
||||
if obj_global_id == sup_obj:
|
||||
self._update_bloc(sup_id, 'start', timestamp)
|
||||
return sup_id
|
||||
|
||||
# create new bloc
|
||||
new_bloc = self._add_bloc(obj_global_id, timestamp)
|
||||
return new_bloc
|
||||
|
||||
# inf_pos == 'start' and sup_pos == 'end'
|
||||
# else raise error ???
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC):
|
|||
if crypto.is_valid_address():
|
||||
# print(address)
|
||||
is_valid_address = True
|
||||
crypto.add(date, item_id)
|
||||
crypto.add(date, item)
|
||||
|
||||
# Check private key
|
||||
if is_valid_address:
|
||||
|
|
|
|||
|
|
@ -210,18 +210,18 @@ class PgpDump(AbstractModule):
|
|||
date = item.get_date()
|
||||
for key in self.keys:
|
||||
pgp = Pgps.Pgp(key, 'key')
|
||||
pgp.add(date, self.item_id)
|
||||
pgp.add(date, item)
|
||||
print(f' key: {key}')
|
||||
for name in self.names:
|
||||
pgp = Pgps.Pgp(name, 'name')
|
||||
pgp.add(date, self.item_id)
|
||||
pgp.add(date, item)
|
||||
print(f' name: {name}')
|
||||
self.tracker_term.compute(name, obj_type='pgp', subtype='name')
|
||||
self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
|
||||
self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
|
||||
for mail in self.mails:
|
||||
pgp = Pgps.Pgp(mail, 'mail')
|
||||
pgp.add(date, self.item_id)
|
||||
pgp.add(date, item)
|
||||
print(f' mail: {mail}')
|
||||
self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
|
||||
self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ class Telegram(AbstractModule):
|
|||
user_id = dict_url.get('username')
|
||||
if user_id:
|
||||
username = Username(user_id, 'telegram')
|
||||
username.add(item_date, item.id)
|
||||
username.add(item_date, item)
|
||||
print(f'username: {user_id}')
|
||||
invite_hash = dict_url.get('invite_hash')
|
||||
if invite_hash:
|
||||
|
|
@ -73,7 +73,7 @@ class Telegram(AbstractModule):
|
|||
user_id = dict_url.get('username')
|
||||
if user_id:
|
||||
username = Username(user_id, 'telegram')
|
||||
username.add(item_date, item.id)
|
||||
username.add(item_date, item)
|
||||
print(f'username: {user_id}')
|
||||
invite_hash = dict_url.get('invite_hash')
|
||||
if invite_hash:
|
||||
|
|
|
|||
|
|
@ -120,14 +120,26 @@
|
|||
|
||||
<div class="chat-message-left pb-0">
|
||||
<div>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['username']['id'] }}" width="40" height="40">
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
|
||||
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
|
||||
</div>
|
||||
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">{{ mess['username']['id'] }}</div>
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['user-account']['username'] %}
|
||||
{{ mess['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if mess['reply_to'] %}
|
||||
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">{{ mess['reply_to']['username']['id'] }}</div>
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['reply_to']['user-account']['username'] %}
|
||||
{{ mess['reply_to']['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['reply_to']['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
|
||||
{% for tag in mess['reply_to']['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
|
|
|
|||
Loading…
Reference in a new issue