diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 24b9bcb8..cc1e3ea8 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -56,6 +56,8 @@ class FeederImporter(AbstractImporter): feeders = [f[:-3] for f in os.listdir(feeder_dir) if os.path.isfile(os.path.join(feeder_dir, f))] self.feeders = {} for feeder in feeders: + if feeder == 'abstract_chats_feeder': + continue print(feeder) part = feeder.split('.')[-1] # import json importer class diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 5ef58b5f..8764aa8b 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from importer.feeders.Default import DefaultFeeder +from importer.feeders.abstract_chats_feeder import AbstractChatFeeder from lib.ConfigLoader import ConfigLoader from lib.objects import ail_objects from lib.objects.Chats import Chat @@ -24,115 +24,15 @@ from lib.objects import UsersAccount from lib.objects.Usernames import Username import base64 -import io -import gzip -# TODO remove compression ??? -def gunzip_bytes_obj(bytes_obj): - gunzipped_bytes_obj = None - try: - in_ = io.BytesIO() - in_.write(bytes_obj) - in_.seek(0) - - with gzip.GzipFile(fileobj=in_, mode='rb') as fo: - gunzipped_bytes_obj = fo.read() - except Exception as e: - print(f'Global; Invalid Gzip file: {e}') - - return gunzipped_bytes_obj - -class TelegramFeeder(DefaultFeeder): +class TelegramFeeder(AbstractChatFeeder): def __init__(self, json_data): - super().__init__(json_data) - self.name = 'telegram' + super().__init__('telegram', json_data) - def get_obj(self): # TODO handle others objects -> images, pdf, ... - # Get message date - timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP - # if self.json_data['meta'].get('date'): - # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) - # date = date.strftime('%Y/%m/%d') - # else: - # date = datetime.date.today().strftime("%Y/%m/%d") - chat_id = str(self.json_data['meta']['chat']['id']) - message_id = str(self.json_data['meta']['id']) - obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) - obj_id = f'message:telegram:{obj_id}' - self.obj = ail_objects.get_obj_from_global_id(obj_id) - return self.obj + # def get_obj(self):. + # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + # obj_id = f'message:telegram:{obj_id}' + # self.obj = ail_objects.get_obj_from_global_id(obj_id) + # return self.obj - def process_meta(self): - """ - Process JSON meta field. - """ - # message chat - meta = self.json_data['meta'] - mess_id = self.json_data['meta']['id'] - if meta.get('reply_to'): - reply_to_id = int(meta['reply_to']) - else: - reply_to_id = None - - timestamp = meta['date']['timestamp'] - date = datetime.datetime.fromtimestamp(timestamp) - date = date.strftime('%Y%m%d') - - if self.json_data.get('translation'): - translation = self.json_data['translation'] - else: - translation = None - decoded = base64.standard_b64decode(self.json_data['data']) - content = gunzip_bytes_obj(decoded) - message = Messages.create(self.obj.id, content, translation=translation) - - if meta.get('chat'): - chat = Chat(meta['chat']['id'], 'telegram') - - if meta['chat'].get('username'): - chat_username = Username(meta['chat']['username'], 'telegram') - chat.update_username_timeline(chat_username.get_global_id(), timestamp) - - # Chat---Message - chat.add(date) - chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id) - else: - chat = None - - # message sender - if meta.get('sender'): # TODO handle message channel forward - check if is user - user_id = meta['sender']['id'] - user_account = UsersAccount.UserAccount(user_id, 'telegram') - # UserAccount---Message - user_account.add(date, obj=message) - # UserAccount---Chat - user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) - - if meta['sender'].get('firstname'): - user_account.set_first_name(meta['sender']['firstname']) - if meta['sender'].get('lastname'): - user_account.set_last_name(meta['sender']['lastname']) - if meta['sender'].get('phone'): - user_account.set_phone(meta['sender']['phone']) - - if meta['sender'].get('username'): - username = Username(meta['sender']['username'], 'telegram') - # TODO timeline or/and correlation ???? - user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - user_account.update_username_timeline(username.get_global_id(), timestamp) - - # Username---Message - username.add(date) # TODO # correlation message ??? - - # if chat: # TODO Chat---Username correlation ??? - # # Chat---Username - # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - - # if meta.get('fwd_from'): - # if meta['fwd_from'].get('post_author') # user first name - - # TODO reply threads ???? - # message edit ???? - - return None diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py new file mode 100755 index 00000000..dfc559b7 --- /dev/null +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Abstract Chat JSON Feeder Importer Module +================ + +Process Feeder Json (example: Twitter feeder) + +""" +import datetime +import os +import sys + +from abc import abstractmethod, ABC + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from importer.feeders.Default import DefaultFeeder +from lib.objects.Chats import Chat +from lib.objects import ChatSubChannels +from lib.objects import Messages +from lib.objects import UsersAccount +from lib.objects.Usernames import Username +from lib import chats_viewer + +import base64 +import io +import gzip + +# TODO remove compression ??? +def _gunzip_bytes_obj(bytes_obj): + gunzipped_bytes_obj = None + try: + in_ = io.BytesIO() + in_.write(bytes_obj) + in_.seek(0) + + with gzip.GzipFile(fileobj=in_, mode='rb') as fo: + gunzipped_bytes_obj = fo.read() + except Exception as e: + print(f'Global; Invalid Gzip file: {e}') + + return gunzipped_bytes_obj + +class AbstractChatFeeder(DefaultFeeder, ABC): + + def __init__(self, name, json_data): + super().__init__(json_data) + self.obj = None + self.name = name + + def get_chat_protocol(self): # TODO # # # # # # # # # # # # # + return self.name + + def get_chat_network(self): + self.json_data['meta'].get('network', None) + + def get_chat_address(self): + self.json_data['meta'].get('address', None) + + def get_chat_instance_uuid(self): + chat_instance_uuid = chats_viewer.create_chat_service_instance(self.get_chat_protocol(), + network=self.get_chat_network(), + address=self.get_chat_address()) + # TODO SET + return chat_instance_uuid + + def get_chat_id(self): # TODO RAISE ERROR IF NONE + return self.json_data['meta']['chat']['id'] + + def get_channel_id(self): + pass + + def get_subchannels(self): + pass + + def get_thread_id(self): + pass + + def get_message_timestamp(self): + return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + # if self.json_data['meta'].get('date'): + # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) + # date = date.strftime('%Y/%m/%d') + # else: + # date = datetime.date.today().strftime("%Y/%m/%d") + + def get_message_date_timestamp(self): + timestamp = self.get_message_timestamp() + date = datetime.datetime.fromtimestamp(timestamp) + date = date.strftime('%Y%m%d') + return date, timestamp + + def get_message_sender_id(self): + return self.json_data['meta']['sender']['id'] + + def get_message_reply(self): + return self.json_data['meta'].get('reply_to') # TODO change to reply ??? + + def get_message_reply_id(self): + return self.json_data['meta'].get('reply_to', None) + + def get_message_content(self): + decoded = base64.standard_b64decode(self.json_data['data']) + return _gunzip_bytes_obj(decoded) + + def get_obj(self): # TODO handle others objects -> images, pdf, ... + #### TIMESTAMP #### + timestamp = self.get_message_timestamp() + + #### Create Object ID #### + chat_id = str(self.json_data['meta']['chat']['id']) + message_id = str(self.json_data['meta']['id']) + # channel id + # thread id + + obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) + self.obj = Messages.Message(obj_id) + return self.obj + + def process_chat(self, message, date, timestamp, reply_id=None): # TODO threads + meta = self.json_data['meta']['chat'] + chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) + chat.add(date) # TODO ### Dynamic subtype ??? + + if meta.get('name'): + chat.set_name(meta['name']) + + if meta.get('date'): # TODO check if already exists + chat.set_created_at(int(meta['date']['timestamp'])) + + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + chat.update_username_timeline(username.get_global_id(), timestamp) + + if meta.get('subchannel'): + subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id) + chat.add_children(obj_global_id=subchannel.get_global_id()) + else: + chat.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id) + + # if meta.get('subchannels'): # TODO Update icon + names + + return chat + + # def process_subchannels(self): + # pass + + def process_subchannel(self, message, date, timestamp, reply_id=None): # TODO CREATE DATE + meta = self.json_data['meta']['chat']['subchannel'] + subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid()) + subchannel.add(date) + + if meta.get('date'): # TODO check if already exists + subchannel.set_created_at(int(meta['date']['timestamp'])) + + if meta.get('name'): + subchannel.set_name(meta['name']) + # subchannel.update_name(meta['name'], timestamp) # TODO ################# + + subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id) + return subchannel + + def process_sender(self, date, timestamp): + meta = self.json_data['meta']['sender'] + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) + + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + # TODO timeline or/and correlation ???? + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + user_account.update_username_timeline(username.get_global_id(), timestamp) + + # Username---Message + username.add(date) # TODO # correlation message ??? + + # ADDITIONAL METAS + if meta.get('firstname'): + user_account.set_first_name(meta['firstname']) + if meta.get('lastname'): + user_account.set_last_name(meta['lastname']) + if meta.get('phone'): + user_account.set_phone(meta['phone']) + + return user_account + + # Create abstract class: -> new API endpoint ??? => force field, check if already imported ? + # 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default + # 2) Create/Get CHAT ID - Done + # 3) Create/Get Channel IF is in channel + # 4) Create/Get Thread IF is in thread + # 5) Create/Update Username and User-account - Done + def process_meta(self): # TODO CHECK MANDATORY FIELDS + """ + Process JSON meta filed. + """ + # meta = self.get_json_meta() + + date, timestamp = self.get_message_date_timestamp() + + # REPLY + reply_id = self.get_message_reply_id() + + # TODO Translation + + # Content + content = self.get_message_content() + + message = Messages.create(self.obj.id, content) # TODO translation + + # CHAT + chat = self.process_chat(message, date, timestamp, reply_id=reply_id) + + # SENDER # TODO HANDLE NULL SENDER + user_account = self.process_sender(date, timestamp) + + # UserAccount---Message + user_account.add(date, obj=message) + # UserAccount---Chat + user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) + + # if chat: # TODO Chat---Username correlation ??? + # # Chat---Username + # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 5e15eabf..4dce4e63 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -13,6 +13,7 @@ from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") +r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', @@ -40,9 +41,11 @@ def get_all_objects(): def get_objects_with_subtypes(): return ['chat', 'cryptocurrency', 'pgp', 'username'] -def get_object_all_subtypes(obj_type): +def get_object_all_subtypes(obj_type): # TODO Dynamic subtype if obj_type == 'chat': - return ['discord', 'jabber', 'telegram'] + return r_object.smembers(f'all_chat:subtypes') + if obj_type == 'chat-subchannel': + return r_object.smembers(f'all_chat-subchannel:subtypes') if obj_type == 'cryptocurrency': return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] if obj_type == 'pgp': diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py new file mode 100755 index 00000000..8c777707 --- /dev/null +++ b/bin/lib/chats_viewer.py @@ -0,0 +1,305 @@ +#!/usr/bin/python3 + +""" +Chats Viewer +=================== + + +""" +import os +import sys +import time +import uuid + + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects import Chats +from lib.objects import ChatSubChannels + +config_loader = ConfigLoader() +r_db = config_loader.get_db_conn("Kvrocks_DB") +r_crawler = config_loader.get_db_conn("Kvrocks_Crawler") +r_cache = config_loader.get_redis_conn("Redis_Cache") + +r_obj = config_loader.get_db_conn("Kvrocks_DB") # TEMP new DB ???? + +# # # # # # # # +# # +# COMMON # +# # +# # # # # # # # + +# TODO ChatDefaultPlatform + +# CHAT(type=chat, subtype=platform, id= chat_id) + +# Channel(type=channel, subtype=platform, id=channel_id) + +# Thread(type=thread, subtype=platform, id=thread_id) + +# Message(type=message, subtype=platform, id=message_id) + + +# Protocol/Platform + + +# class ChatProtocols: # TODO Remove Me +# +# def __init__(self): # name ???? subtype, id ???? +# # discord, mattermost, ... +# pass +# +# def get_chat_protocols(self): +# pass +# +# def get_chat_protocol(self, protocol): +# pass +# +# ################################################################ +# +# def get_instances(self): +# pass +# +# def get_chats(self): +# pass +# +# def get_chats_by_instance(self, instance): +# pass +# +# +# class ChatNetwork: # uuid or protocol +# def __init__(self, network='default'): +# self.id = network +# +# def get_addresses(self): +# pass +# +# +# class ChatServerAddress: # uuid or protocol + network +# def __init__(self, address='default'): +# self.id = address + +# map uuid -> type + field + +# TODO option last protocol/ imported messages/chat -> unread mode ???? + +# # # # # # # # # +# # +# PROTOCOLS # IRC, discord, mattermost, ... +# # +# # # # # # # # # TODO icon => UI explorer by protocol + network + instance + +def get_chat_protocols(): + return r_obj.smembers(f'chat:protocols') + +def get_chat_protocols_meta(): + metas = [] + for protocol_id in get_chat_protocols(): + protocol = ChatProtocol(protocol_id) + metas.append(protocol.get_meta(options={'icon'})) + return metas + +class ChatProtocol: # TODO first seen last seen ???? + nb by day ???? + def __init__(self, protocol): + self.id = protocol + + def exists(self): + return r_db.exists(f'chat:protocol:{self.id}') + + def get_networks(self): + return r_db.smembers(f'chat:protocol:{self.id}') + + def get_nb_networks(self): + return r_db.scard(f'chat:protocol:{self.id}') + + def get_icon(self): + if self.id == 'discord': + icon = {'style': 'fab', 'icon': 'fa-discord'} + elif self.id == 'telegram': + icon = {'style': 'fab', 'icon': 'fa-telegram'} + else: + icon = {} + return icon + + def get_meta(self, options=set()): + meta = {'id': self.id} + if 'icon' in options: + meta['icon'] = self.get_icon() + return meta + + # def get_addresses(self): + # pass + # + # def get_instances_uuids(self): + # pass + + +# # # # # # # # # # # # # # +# # +# ChatServiceInstance # +# # +# # # # # # # # # # # # # # + +# uuid -> protocol + network + server +class ChatServiceInstance: + def __init__(self, instance_uuid): + self.uuid = instance_uuid + + def exists(self): + return r_obj.exists(f'chatSerIns:{self.uuid}') + + def get_protocol(self): # return objects ???? + return r_obj.hget(f'chatSerIns:{self.uuid}', 'protocol') + + def get_network(self): # return objects ???? + network = r_obj.hget(f'chatSerIns:{self.uuid}', 'network') + if network: + return network + + def get_address(self): # return objects ???? + address = r_obj.hget(f'chatSerIns:{self.uuid}', 'address') + if address: + return address + + def get_meta(self, options=set()): + meta = {'uuid': self.uuid, + 'protocol': self.get_protocol(), + 'network': self.get_network(), + 'address': self.get_address()} + if 'chats' in options: + meta['chats'] = [] + for chat_id in self.get_chats(): + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'})) + return meta + + def get_nb_chats(self): + return Chats.Chats().get_nb_ids_by_subtype(self.uuid) + + def get_chats(self): + return Chats.Chats().get_ids_by_subtype(self.uuid) + +def get_chat_service_instances(): + return r_obj.smembers(f'chatSerIns:all') + +def get_chat_service_instance_uuid(protocol, network, address): + if not network: + network = '' + if not address: + address = '' + return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address) + +def get_chat_service_instance(protocol, network, address): + instance_uuid = get_chat_service_instance_uuid(protocol, network, address) + if instance_uuid: + return ChatServiceInstance(instance_uuid) + +def create_chat_service_instance(protocol, network=None, address=None): + instance_uuid = get_chat_service_instance_uuid(protocol, network, address) + if instance_uuid: + return instance_uuid + else: + if not network: + network = '' + if not address: + address = '' + instance_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f'{protocol}|{network}|{address}')) + r_obj.sadd(f'chatSerIns:all', instance_uuid) + + # map instance - uuid + r_obj.hset(f'map:chatSerIns:{protocol}:{network}', address, instance_uuid) + + r_obj.hset(f'chatSerIns:{instance_uuid}', 'protocol', protocol) + if network: + r_obj.hset(f'chatSerIns:{instance_uuid}', 'network', network) + if address: + r_obj.hset(f'chatSerIns:{instance_uuid}', 'address', address) + + # protocols + r_obj.sadd(f'chat:protocols', protocol) # TODO first seen / last seen + + # protocol -> network + r_obj.sadd(f'chat:protocol:networks:{protocol}', network) + + return instance_uuid + + + + + # INSTANCE ===> CHAT IDS + + + + + + # protocol -> instance_uuids => for protocol->networks -> protocol+network => HGETALL + # protocol+network -> instance_uuids => HGETALL + + # protocol -> networks ???default??? or '' + + # -------------------------------------------------------- + # protocol+network -> addresses => HKEYS + # protocol+network+addresse => HGET + + +# Chat -> subtype=uuid, id = chat id + + +# instance_uuid -> chat id + + +# protocol - uniq ID +# protocol + network -> uuid ???? +# protocol + network + address -> uuid + +####################################################################################### + +def get_subchannels_meta_from_global_id(subchannels): + meta = [] + for sub in subchannels: + _, instance_uuid, sub_id = sub.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid) + meta.append(subchannel.get_meta({'nb_messages'})) + return meta + +def api_get_chat_service_instance(chat_instance_uuid): + chat_instance = ChatServiceInstance(chat_instance_uuid) + if not chat_instance.exists(): + return {"status": "error", "reason": "Unknown uuid"}, 404 + return chat_instance.get_meta({'chats'}), 200 + +def api_get_chat(chat_id, chat_instance_uuid): + chat = Chats.Chat(chat_id, chat_instance_uuid) + if not chat.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + meta = chat.get_meta({'img', 'subchannels', 'username'}) + if meta['subchannels']: + print(meta['subchannels']) + meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) + return meta, 200 + +def api_get_subchannel(chat_id, chat_instance_uuid): + subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) + if not subchannel.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + meta = subchannel.get_meta({'img', 'nb_messages'}) + meta['messages'], meta['tags_messages'] = subchannel.get_messages() + return meta, 200 + +# # # # # # # # # # LATER +# # +# ChatCategory # +# # +# # # # # # # # # # + + +if __name__ == '__main__': + r = get_chat_service_instances() + print(r) + r = ChatServiceInstance(r.pop()) + print(r.get_meta({'chats'})) + # r = get_chat_protocols() + # print(r) \ No newline at end of file diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py new file mode 100755 index 00000000..15521196 --- /dev/null +++ b/bin/lib/objects/ChatSubChannels.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects + +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib.timeline_engine import Timeline + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class ChatSubChannel(AbstractChatObject): + """ + AIL Chat Object. (strings) + """ + + # ID -> / subtype = chat_instance_uuid + def __init__(self, id, subtype): + super(ChatSubChannel, self).__init__('chat-subchannel', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + # TODO TIME LAST MESSAGES + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['tags'] = self.get_tags(r_list=True) + meta['name'] = self.get_name() + if 'img' in options: + meta['sub'] = self.get_img() + if 'nb_messages': + meta['nb_messages'] = self.get_nb_messages() + return meta + + def get_misp_object(self): + # obj_attrs = [] + # if self.subtype == 'telegram': + # obj = MISPObject('telegram-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # elif self.subtype == 'twitter': + # obj = MISPObject('twitter-account', standalone=True) + # obj_attrs.append(obj.add_attribute('name', value=self.id)) + # + # else: + # obj = MISPObject('user-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # first_seen = self.get_first_seen() + # last_seen = self.get_last_seen() + # if first_seen: + # obj.first_seen = first_seen + # if last_seen: + # obj.last_seen = last_seen + # if not first_seen or not last_seen: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + # + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + def _get_timeline_name(self): + return Timeline(self.get_global_id(), 'username') + + def update_name(self, name, timestamp): + self._get_timeline_name().add_timestamp(timestamp, name) + + + # TODO # # # # # # # # # # # + def get_users(self): + pass + + #### Categories #### + + #### Threads #### + + #### Messages #### TODO set parents + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + +class ChatSubChannels(AbstractChatObjects): + def __init__(self): + super().__init__('chat-subchannels') + +# if __name__ == '__main__': +# chat = Chat('test', 'telegram') +# r = chat.get_messages() +# print(r) diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py new file mode 100755 index 00000000..ac78be2a --- /dev/null +++ b/bin/lib/objects/ChatThreads.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib.timeline_engine import Timeline + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Chat(AbstractSubtypeObject): # TODO # ID == username ????? + """ + AIL Chat Object. (strings) + """ + + def __init__(self, id, subtype): + super(Chat, self).__init__('chat-thread', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + if 'username': + meta['username'] = self.get_username() + return meta + + def get_misp_object(self): + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + #### Messages #### TODO set parents + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + + +if __name__ == '__main__': + chat = Chat('test', 'telegram') + r = chat.get_messages() + print(r) diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index bb27413d..bf73e95b 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -15,6 +15,9 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects + + from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.data_retention_engine import update_obj_date from lib.objects import ail_objects @@ -33,19 +36,14 @@ config_loader = None ################################################################################ ################################################################################ -class Chat(AbstractSubtypeObject): # TODO # ID == username ????? +class Chat(AbstractChatObject): """ - AIL Chat Object. (strings) + AIL Chat Object. """ def __init__(self, id, subtype): super(Chat, self).__init__('chat', id, subtype) - # def get_ail_2_ail_payload(self): - # payload = {'raw': self.get_gzip_content(b64=True), - # 'compress': 'gzip'} - # return payload - # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ def delete(self): # # TODO: @@ -74,9 +72,16 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_meta(self, options=set()): meta = self._get_meta(options=options) - meta['id'] = self.id - meta['subtype'] = self.subtype + meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) + if 'img': + meta['icon'] = self.get_img() + if 'username' in options: + meta['username'] = self.get_username() + if 'subchannels' in options: + meta['subchannels'] = self.get_subchannels() + if 'nb_subchannels': + meta['nb_subchannels'] = self.get_nb_subchannels() return meta def get_misp_object(self): @@ -112,11 +117,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? ############################################################################ ############################################################################ - # others optional metas, ... -> # TODO ALL meta in hset - - def get_name(self): # get username ???? - pass - # users that send at least a message else participants/spectator # correlation created by messages def get_users(self): @@ -138,22 +138,22 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) + #### ChatSubChannels #### + + + #### Categories #### + + #### Threads #### + + #### Messages #### TODO set parents # def get_last_message_id(self): # # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') - def get_obj_message_id(self, obj_id): - if obj_id.endswith('.gz'): - obj_id = obj_id[:-3] - return int(obj_id.split('_')[-1]) - def _get_message_timestamp(self, obj_global_id): return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id) - def _get_messages(self): - return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) - def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): obj = ail_objects.get_obj_from_global_id(obj_global_id) mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) @@ -179,26 +179,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') return mess_dict - - def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies - start = 0 - stop = -1 - # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}') - - # TODO chat without username ???? -> chat ID ???? - - messages = {} - curr_date = None - for message in self._get_messages(): - date = datetime.fromtimestamp(message[1]) - date_day = date.strftime('%Y/%m/%d') - if date_day != curr_date: - messages[date_day] = [] - curr_date = date_day - mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) - messages[date_day].append(mess_dict) - return messages - # Zset with ID ??? id -> item id ??? multiple id == media + text # id -> media id # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ??? @@ -229,43 +209,12 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? # domain = get_item_domain(item_id) # self.add_correlation('domain', '', domain) - # TODO kvrocks exception if key don't exists - def get_obj_by_message_id(self, mess_id): - return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id) - # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn - def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None): - r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id) - r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp}) - if reply_id: - reply_obj = self.get_obj_by_message_id(reply_id) - if reply_obj: - self.add_obj_children(reply_obj, obj_global_id) - else: - self.add_message_cached_reply(reply_id, mess_id) - - # ADD cached replies - for reply_obj in self.get_cached_message_reply(mess_id): - self.add_obj_children(obj_global_id, reply_obj) - - def _get_message_cached_reply(self, message_id): - return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') - - def get_cached_message_reply(self, message_id): - objs_global_id = [] - for mess_id in self._get_message_cached_reply(message_id): - obj_global_id = self.get_obj_by_message_id(mess_id) - if obj_global_id: - objs_global_id.append(obj_global_id) - return objs_global_id - - def add_message_cached_reply(self, reply_to_id, message_id): - r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id) - r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600) - - # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering +class Chats(AbstractChatObjects): + def __init__(self): + super().__init__('chat') # TODO factorize def get_all_subtypes(): @@ -280,28 +229,6 @@ def get_all(): def get_all_by_subtype(subtype): return get_all_id('chat', subtype) -# # TODO FILTER NAME + Key + mail -# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME -# -# return name_to_search -# -# def search_usernames_by_name(name_to_search, subtype, r_pos=False): -# usernames = {} -# # for subtype in subtypes: -# r_name = sanitize_username_name_to_search(name_to_search, subtype) -# if not name_to_search or isinstance(r_name, dict): -# # break -# return usernames -# r_name = re.compile(r_name) -# for user_name in get_all_usernames_by_subtype(subtype): -# res = re.search(r_name, user_name) -# if res: -# usernames[user_name] = {} -# if r_pos: -# usernames[user_name]['hl-start'] = res.start() -# usernames[user_name]['hl-end'] = res.end() -# return usernames - if __name__ == '__main__': chat = Chat('test', 'telegram') diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 2f1ef5de..a1e42c10 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -43,6 +43,10 @@ config_loader = None # /!\ handle null chat and message id -> chat = uuid and message = timestamp ??? +# ID = /// => telegram without channels +# ID = //// +# ID = //// +# ID = ///// class Message(AbstractObject): """ AIL Message Object. (strings) @@ -86,7 +90,7 @@ class Message(AbstractObject): return dirs[-2] def get_message_id(self): # TODO optimize - message_id = self.get_basename().rsplit('_', 1)[1] + message_id = self.get_basename().rsplit('/', 1)[1] # if message_id.endswith('.gz'): # message_id = message_id[:-3] return message_id @@ -97,6 +101,10 @@ class Message(AbstractObject): # chat_id = chat_id[:-3] return chat_id + # TODO get Instance ID + # TODO get channel ID + # TODO get thread ID + def get_user_account(self): user_account = self.get_correlation('user-account') if user_account.get('user-account'): @@ -255,8 +263,16 @@ class Message(AbstractObject): def delete(self): pass -def create_obj_id(source, chat_id, message_id, timestamp): - return f'{source}/{timestamp}/{chat_id}_{message_id}' +def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): + timestamp = int(timestamp) + if channel_id and thread_id: + return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ????? + elif channel_id: + return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}' + elif thread_id: + return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}' + else: + return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}' # TODO Check if already exists # def create(source, chat_id, message_id, timestamp, content, tags=[]): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py new file mode 100755 index 00000000..be8c1397 --- /dev/null +++ b/bin/lib/objects/abstract_chat_object.py @@ -0,0 +1,241 @@ +# -*-coding:UTF-8 -* +""" +Base Class for AIL Objects +""" + +################################## +# Import External packages +################################## +import os +import sys +from abc import ABC + +from datetime import datetime +# from flask import url_for + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects.abstract_subtype_object import AbstractSubtypeObject +from lib.ail_core import get_object_all_subtypes, zscan_iter ################ +from lib.ConfigLoader import ConfigLoader +from lib.objects.Messages import Message +from lib.objects.UsersAccount import UserAccount +from lib.objects.Usernames import Username +from lib.data_retention_engine import update_obj_date + +from packages import Date + +# LOAD CONFIG +config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +config_loader = None + +# # FIXME: SAVE SUBTYPE NAMES ????? + +class AbstractChatObject(AbstractSubtypeObject, ABC): + """ + Abstract Subtype Object + """ + + def __init__(self, obj_type, id, subtype): + """ Abstract for all the AIL object + + :param obj_type: object type (item, ...) + :param id: Object ID + """ + super().__init__(obj_type, id, subtype) + + # get useraccount / username + # get users ? + # timeline name ???? + # info + # created + # last imported/updated + + # TODO get instance + # TODO get protocol + # TODO get network + # TODO get address + + def get_chat(self): # require ail object TODO ## + if self.type != 'chat': + parent = self.get_parent() + obj_type, _ = parent.split(':', 1) + if obj_type == 'chat': + return parent + + def get_subchannels(self): + subchannels = [] + if self.type == 'chat': # category ??? + print(self.get_childrens()) + for obj_global_id in self.get_childrens(): + print(obj_global_id) + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-subchannel': + subchannels.append(obj_global_id) + return subchannels + + def get_nb_subchannels(self): + nb = 0 + if self.type == 'chat': + for obj_global_id in self.get_childrens(): + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-subchannel': + nb += 1 + return nb + + def get_threads(self): + threads = [] + for obj_global_id in self.get_childrens(): + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-thread': + threads.append(obj_global_id) + return threads + + def get_created_at(self): + return self._get_field('created_at') + + def set_created_at(self, timestamp): + self._set_field('created_at', timestamp) + + def get_name(self): + name = self._get_field('name') + if not name: + name = '' + return name + + def set_name(self, name): + self._set_field('name', name) + + def get_img(self): + return self._get_field('img') + + def set_img(self, icon): + self._set_field('img', icon) + + def get_nb_messages(self): + return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}') + + def _get_messages(self): # TODO paginate + return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + + def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message + obj = Message(message[9:]) + mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) + print(mess_dict) + if mess_dict.get('parent') and parent: + mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) + if mess_dict.get('user-account'): + _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3) + print(mess_dict['user-account']) + user_account = UserAccount(user_account_id, user_account_subtype) + mess_dict['user-account'] = {} + mess_dict['user-account']['type'] = user_account.get_type() + mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) + mess_dict['user-account']['id'] = user_account.get_id() + username = user_account.get_username() + if username: + _, username_account_subtype, username_account_id = username.split(':', 3) + username = Username(username_account_id, username_account_subtype).get_default_meta(link=False) + mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? + else: + mess_dict['user-account'] = {'id': 'UNKNOWN'} + + if not mess_datetime: + obj_mess_id = message.get_timestamp() + mess_datetime = datetime.fromtimestamp(obj_mess_id) + mess_dict['date'] = mess_datetime.isoformat(' ') + mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') + return mess_dict + + def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? + # TODO return message meta + tags = {} + messages = {} + curr_date = None + for message in self._get_messages(): + date = datetime.fromtimestamp(message[1]) + date_day = date.strftime('%Y/%m/%d') + if date_day != curr_date: + messages[date_day] = [] + curr_date = date_day + mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object + messages[date_day].append(mess_dict) + + if mess_dict.get('tags'): + for tag in mess_dict['tags']: + if tag not in tags: + tags[tag] = 0 + tags[tag] += 1 + return messages, tags + + # TODO REWRITE ADD OR ADD MESSAGE ???? + # add + # add message + + def get_obj_by_message_id(self, message_id): + return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id) + + def add_message_cached_reply(self, reply_id, message_id): + r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id) + r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600) + + def _get_message_cached_reply(self, message_id): + return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') + + def get_cached_message_reply(self, message_id): + objs_global_id = [] + for mess_id in self._get_message_cached_reply(message_id): + obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION + if obj_global_id: + objs_global_id.append(obj_global_id) + return objs_global_id + + def add_message(self, obj_global_id, message_id, timestamp, reply_id=None): + r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id) + r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)}) + + # MESSAGE REPLY + if reply_id: + reply_obj = self.get_obj_by_message_id(reply_id) # TODO CATCH EXCEPTION + if reply_obj: + self.add_obj_children(reply_obj, obj_global_id) + else: + self.add_message_cached_reply(reply_id, message_id) + + + # get_messages_meta ???? + +# TODO move me to abstract subtype +class AbstractChatObjects(ABC): + def __init__(self, type): + self.type = type + + def add_subtype(self, subtype): + r_object.sadd(f'all_{self.type}:subtypes', subtype) + + def get_subtypes(self): + return r_object.smembers(f'all_{self.type}:subtypes') + + def get_nb_ids_by_subtype(self, subtype): + return r_object.zcard(f'{self.type}_all:{subtype}') + + def get_ids_by_subtype(self, subtype): + print(subtype) + print(f'{self.type}_all:{subtype}') + return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1) + + def get_all_id_iterator_iter(self, subtype): + return zscan_iter(r_object, f'{self.type}_all:{subtype}') + + def get_ids(self): + pass + + def search(self): + pass + + + diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index a3f25216..808e7547 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -289,23 +289,24 @@ class AbstractObject(ABC): def get_parent(self): return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') - def get_children(self): + def get_childrens(self): return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') - def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP if not obj_global_id: if obj_subtype is None: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id) - def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP if not obj_global_id: if obj_subtype is None: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id) + ## others objects ## def add_obj_children(self, parent_global_id, son_global_id): r_object.sadd(f'child:{parent_global_id}', son_global_id) r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id) diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 007f716b..fa0030da 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -88,7 +88,9 @@ class AbstractSubtypeObject(AbstractObject, ABC): def _get_meta(self, options=None): if options is None: options = set() - meta = {'first_seen': self.get_first_seen(), + meta = {'id': self.id, + 'subtype': self.subtype, + 'first_seen': self.get_first_seen(), 'last_seen': self.get_last_seen(), 'nb_seen': self.get_nb_seen()} if 'icon' in options: @@ -150,8 +152,11 @@ class AbstractSubtypeObject(AbstractObject, ABC): # => data Retention + efficient search # # + def _add_subtype(self): + r_object.sadd(f'all_{self.type}:subtypes', self.subtype) def add(self, date, obj=None): + self._add_subtype() self.update_daterange(date) update_obj_date(date, self.type, self.subtype) # daily diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c330443b..97f6dde3 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -50,7 +50,7 @@ from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name from blueprints.objects_etag import objects_etag from blueprints.objects_hhhash import objects_hhhash -from blueprints.objects_chat import objects_chat +from blueprints.chats_explorer import chats_explorer Flask_dir = os.environ['AIL_FLASK'] @@ -108,7 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_hhhash, url_prefix=baseUrl) -app.register_blueprint(objects_chat, url_prefix=baseUrl) +app.register_blueprint(chats_explorer, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py new file mode 100644 index 00000000..ff180a32 --- /dev/null +++ b/var/www/blueprints/chats_explorer.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import chats_viewer + + + +############################################ + +from lib import ail_core +from lib.objects import ail_objects +from lib import chats_viewer +from lib.objects import Chats +from lib.objects import ChatSubChannels + +# ============ BLUEPRINT ============ +chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +# ============= ROUTES ============== + +@chats_explorer.route("/chats/explorer", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_dashboard(): + return + +@chats_explorer.route("chats/explorer/protocols", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_protocols(): + protocols = chats_viewer.get_chat_protocols_meta() + return render_template('chats_protocols.html', protocols=protocols) + +@chats_explorer.route("chats/explorer/instance", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_instance(): + intance_uuid = request.args.get('uuid') + chat_instance = chats_viewer.api_get_chat_service_instance(intance_uuid) + if chat_instance[1] != 200: + return create_json_response(chat_instance[0], chat_instance[1]) + else: + chat_instance = chat_instance[0] + return render_template('chat_instance.html', chat_instance=chat_instance) + +@chats_explorer.route("chats/explorer/chat", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_chat(): + chat_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + chat = chats_viewer.api_get_chat(chat_id, instance_uuid) + if chat[1] != 200: + return create_json_response(chat[0], chat[1]) + else: + chat = chat[0] + return render_template('chat_viewer.html', chat=chat) + +@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) +@login_required +@login_read_only +def objects_subchannel_messages(): + subchannel_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid) + if subchannel[1] != 200: + return create_json_response(subchannel[0], subchannel[1]) + else: + subchannel = subchannel[0] + return render_template('SubChannelMessages.html', subchannel=subchannel) + +@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) +@login_required +@login_read_only +def objects_message(): + message_id = request.args.get('id') + message = chats_viewer.api_get_message(message_id) + if message[1] != 200: + return create_json_response(message[0], message[1]) + else: + message = message[0] + return render_template('ChatMessage.html', message=message) + +############################################################################################# +############################################################################################# +############################################################################################# + + +@chats_explorer.route("/objects/chat/messages", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + chat = request.args.get('id') + subtype = request.args.get('subtype') + chat = Chats.Chat(chat, subtype) + if chat.exists(): + messages, mess_tags = chat.get_messages() + print(messages) + print(chat.get_subchannels()) + meta = chat.get_meta({'icon', 'username'}) + if meta.get('username'): + meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta() + print(meta) + return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label) + else: + return abort(404) diff --git a/var/www/blueprints/objects_chat.py b/var/www/blueprints/objects_chat.py deleted file mode 100644 index 8a1db11f..00000000 --- a/var/www/blueprints/objects_chat.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... -''' - -import os -import sys -import json - -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file -from flask_login import login_required, current_user - -# Import Role_Manager -from Role_Manager import login_admin, login_analyst, login_read_only - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ail_core -from lib.objects import abstract_subtype_object -from lib.objects import ail_objects -from lib.objects import Chats -from packages import Date - -# ============ BLUEPRINT ============ -objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat')) - -# ============ VARIABLES ============ -bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] - -def create_json_response(data, status_code): - return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code - -# ============ FUNCTIONS ============ - -# ============= ROUTES ============== - - -@objects_chat.route("/objects/chat/messages", methods=['GET']) -@login_required -@login_read_only -def objects_dashboard_chat(): - chat = request.args.get('id') - subtype = request.args.get('subtype') - chat = Chats.Chat(chat, subtype) - if chat.exists(): - messages = chat.get_messages() - meta = chat.get_meta({'icon'}) - print(meta) - return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label) - else: - return abort(404) - - - diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html new file mode 100644 index 00000000..32aae48e --- /dev/null +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -0,0 +1,209 @@ + + + + + Chat Messages - AIL + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+

{{ meta["id"] }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + + +
    Object subtypeFirst seenLast seenUsernameNb seen
    + + + + {{ meta["icon"]["icon"] }} + + + {{ meta["subtype"] }} + {{ meta['first_seen'] }}{{ meta['last_seen'] }} + {% if 'username' in meta %} + {{ meta['username']['id'] }} + {% endif %} + {{ meta['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
  • +
    +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
+
+ {% for date in messages %} + {{ date }} + {% endfor %} +
+
+ +
+
+ +

{{ date }}

+ +
+
+ {{ mess['user-account']['id'] }} +
{{ mess['hour'] }}
+
+
+
+ {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
+ {% if mess['reply_to'] %} +
+
+ {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} +
+ 
{{ mess['reply_to']['content'] }}
+ {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
{{ mess['reply_to']['date'] }}
+{#
#} +{# #} +{# #} +{#
#} +
+ {% endif %} + 
{{ mess['content'] }}
+ {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
+ + +
+
+
+ +
+
+ +
+ +
+
+ + + + + + diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/chats_explorer/ChatMessages.html similarity index 90% rename from var/www/templates/objects/chat/ChatMessages.html rename to var/www/templates/chats_explorer/ChatMessages.html index dd99b250..729a132c 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/chats_explorer/ChatMessages.html @@ -58,6 +58,7 @@ Object subtype First seen Last seen + Username Nb seen @@ -74,6 +75,11 @@ {{ meta['first_seen'] }} {{ meta['last_seen'] }} + + {% if 'username' in meta %} + {{ meta['username']['id'] }} + {% endif %} + {{ meta['nb_seen'] }} @@ -111,11 +117,23 @@ + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
+
+ {% for date in messages %} + {{ date }} + {% endfor %} +
+
+
{% for date in messages %} -

{{ date }}

+

{{ date }}

{% for mess in messages[date] %}
diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html new file mode 100644 index 00000000..846e9cbd --- /dev/null +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -0,0 +1,211 @@ + + + + + Sub-Channel Messages - AIL + + + + + + +{# #} + + + + + + + +{# + #} + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+

{{ subchannel["id"] }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + + +
    Chat InstanceFirst seenLast seenUsernameNb Messages
    + {{ subchannel["subtype"] }} + {{ subchannel['first_seen'] }}{{ subchannel['last_seen'] }} + {% if 'username' in subchannel %} + {{ subchannel['username'] }} + {% endif %} + {{ subchannel['nb_messages'] }}
    +
    +
    +
    • +
  • +
    +
    + Tags: + {% for tag in subchannel['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
+
+ {% for date in messages %} + {{ date }} + {% endfor %} +
+
+ +
+
+ + {% for date in subchannel['messages'] %} +

{{ date }}

+ {% for mess in subchannel['messages'][date] %} + +
+
+ {{ mess['user-account']['id'] }} +
{{ mess['hour'] }}
+
+
+
+ {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
+ {% if mess['reply_to'] %} +
+
+ {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} +
+ 
{{ mess['reply_to']['content'] }}
+ {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
{{ mess['reply_to']['date'] }}
+{#
#} +{# #} +{# #} +{#
#} +
+ {% endif %} + 
{{ mess['content'] }}
+ {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
+ + +
+
+
+ + {% endfor %} +
+ {% endfor %} + +
+
+ +
+ +
+
+ + + + + + diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html new file mode 100644 index 00000000..5ddbc136 --- /dev/null +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -0,0 +1,123 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+ +
+

{{ chat_instance["protocol"] }} :

+
    +
  • + + + + + + + + + + + + + + + + + +
    Chat InstanceNetworkAddressNb Chats
    + {{ chat_instance["uuid"] }} + {% if chat_instance["network"] %}{{ chat_instance["network"]}}{% endif %}{% if chat_instance["address"] %}{{ chat_instance["address"] }}{% endif %}{{ chat_instance["chats"] | length }}
    +
    • +
+ +
+
+ + + + + + + + + + + + + + {% for chat in chat_instance["chats"] %} + + + + + + + + + {% endfor %} + +
IconNameIDFirst SeenLast SeenNB Chats
{{ chat['name'] }}{{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }}{{ chat['nb_subchannels'] }}
+ +
+ +
+
+ + + + + + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html new file mode 100644 index 00000000..97059d95 --- /dev/null +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -0,0 +1,133 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+ +
+

{{ chat["username"] }} {{ chat["id"] }} :

+
    +
  • + + + + + + + + + + + + + + + + + + + + + +
    IconNameIDFirst SeenLast SeenNB Sub-Channels
    {{ chat['name'] }}{{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }}{{ chat['nb_subchannels'] }}
    +
    • +
+ +
+
+ + {% if chat['subchannels'] %} +

Sub-Channels:

+ + + + + + + + + + + + + {% for meta in chat["subchannels"] %} + + + + + + + + + {% endfor %} + +
IconNameIDFirst SeenLast SeenNB Messages
+ {{ meta['id'] }} + {{ meta['name'] }}{{ meta['id'] }}{{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['nb_messages'] }}
+ + {% endif %} + +
+ +
+
+ + + + + + diff --git a/var/www/templates/chats_explorer/chats_instance.html b/var/www/templates/chats_explorer/chats_instance.html new file mode 100644 index 00000000..f7b56e39 --- /dev/null +++ b/var/www/templates/chats_explorer/chats_instance.html @@ -0,0 +1,80 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ + {% if protocols %} +
+ {% for meta in protocols %} + +
+

+ {% if meta['icon'] %} + + {% endif %} + {{ meta['id'] }} +

+
+ + {% endfor %} +
+ + {% else %} +

No Protocol/Chats Imported

+ {% endif %} + +
+ +
+
+ + + + + + diff --git a/var/www/templates/chats_explorer/chats_protocols.html b/var/www/templates/chats_explorer/chats_protocols.html new file mode 100644 index 00000000..f7b56e39 --- /dev/null +++ b/var/www/templates/chats_explorer/chats_protocols.html @@ -0,0 +1,80 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ + {% if protocols %} +
+ {% for meta in protocols %} + +
+

+ {% if meta['icon'] %} + + {% endif %} + {{ meta['id'] }} +

+
+ + {% endfor %} +
+ + {% else %} +

No Protocol/Chats Imported

+ {% endif %} + +
+ +
+
+ + + + + +