mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [chats] add chats explorer v0
This commit is contained in:
terrtia
parent
c5cef5fd00
commit
9125119764
22 changed files with 2109 additions and 276 deletions
|
|
@ -56,6 +56,8 @@ class FeederImporter(AbstractImporter):
|
|||
feeders = [f[:-3] for f in os.listdir(feeder_dir) if os.path.isfile(os.path.join(feeder_dir, f))]
|
||||
self.feeders = {}
|
||||
for feeder in feeders:
|
||||
if feeder == 'abstract_chats_feeder':
|
||||
continue
|
||||
print(feeder)
|
||||
part = feeder.split('.')[-1]
|
||||
# import json importer class
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from importer.feeders.Default import DefaultFeeder
|
||||
from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects import ail_objects
|
||||
from lib.objects.Chats import Chat
|
||||
|
|
@ -24,115 +24,15 @@ from lib.objects import UsersAccount
|
|||
from lib.objects.Usernames import Username
|
||||
|
||||
import base64
|
||||
import io
|
||||
import gzip
|
||||
|
||||
# TODO remove compression ???
|
||||
def gunzip_bytes_obj(bytes_obj):
|
||||
gunzipped_bytes_obj = None
|
||||
try:
|
||||
in_ = io.BytesIO()
|
||||
in_.write(bytes_obj)
|
||||
in_.seek(0)
|
||||
|
||||
with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
|
||||
gunzipped_bytes_obj = fo.read()
|
||||
except Exception as e:
|
||||
print(f'Global; Invalid Gzip file: {e}')
|
||||
|
||||
return gunzipped_bytes_obj
|
||||
|
||||
class TelegramFeeder(DefaultFeeder):
|
||||
class TelegramFeeder(AbstractChatFeeder):
|
||||
|
||||
def __init__(self, json_data):
|
||||
super().__init__(json_data)
|
||||
self.name = 'telegram'
|
||||
super().__init__('telegram', json_data)
|
||||
|
||||
def get_obj(self): # TODO handle others objects -> images, pdf, ...
|
||||
# Get message date
|
||||
timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
|
||||
# if self.json_data['meta'].get('date'):
|
||||
# date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
|
||||
# date = date.strftime('%Y/%m/%d')
|
||||
# else:
|
||||
# date = datetime.date.today().strftime("%Y/%m/%d")
|
||||
chat_id = str(self.json_data['meta']['chat']['id'])
|
||||
message_id = str(self.json_data['meta']['id'])
|
||||
obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
|
||||
obj_id = f'message:telegram:{obj_id}'
|
||||
self.obj = ail_objects.get_obj_from_global_id(obj_id)
|
||||
return self.obj
|
||||
# def get_obj(self):.
|
||||
# obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
|
||||
# obj_id = f'message:telegram:{obj_id}'
|
||||
# self.obj = ail_objects.get_obj_from_global_id(obj_id)
|
||||
# return self.obj
|
||||
|
||||
def process_meta(self):
|
||||
"""
|
||||
Process JSON meta field.
|
||||
"""
|
||||
# message chat
|
||||
meta = self.json_data['meta']
|
||||
mess_id = self.json_data['meta']['id']
|
||||
if meta.get('reply_to'):
|
||||
reply_to_id = int(meta['reply_to'])
|
||||
else:
|
||||
reply_to_id = None
|
||||
|
||||
timestamp = meta['date']['timestamp']
|
||||
date = datetime.datetime.fromtimestamp(timestamp)
|
||||
date = date.strftime('%Y%m%d')
|
||||
|
||||
if self.json_data.get('translation'):
|
||||
translation = self.json_data['translation']
|
||||
else:
|
||||
translation = None
|
||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||
content = gunzip_bytes_obj(decoded)
|
||||
message = Messages.create(self.obj.id, content, translation=translation)
|
||||
|
||||
if meta.get('chat'):
|
||||
chat = Chat(meta['chat']['id'], 'telegram')
|
||||
|
||||
if meta['chat'].get('username'):
|
||||
chat_username = Username(meta['chat']['username'], 'telegram')
|
||||
chat.update_username_timeline(chat_username.get_global_id(), timestamp)
|
||||
|
||||
# Chat---Message
|
||||
chat.add(date)
|
||||
chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
|
||||
else:
|
||||
chat = None
|
||||
|
||||
# message sender
|
||||
if meta.get('sender'): # TODO handle message channel forward - check if is user
|
||||
user_id = meta['sender']['id']
|
||||
user_account = UsersAccount.UserAccount(user_id, 'telegram')
|
||||
# UserAccount---Message
|
||||
user_account.add(date, obj=message)
|
||||
# UserAccount---Chat
|
||||
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
|
||||
|
||||
if meta['sender'].get('firstname'):
|
||||
user_account.set_first_name(meta['sender']['firstname'])
|
||||
if meta['sender'].get('lastname'):
|
||||
user_account.set_last_name(meta['sender']['lastname'])
|
||||
if meta['sender'].get('phone'):
|
||||
user_account.set_phone(meta['sender']['phone'])
|
||||
|
||||
if meta['sender'].get('username'):
|
||||
username = Username(meta['sender']['username'], 'telegram')
|
||||
# TODO timeline or/and correlation ????
|
||||
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
user_account.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
||||
# Username---Message
|
||||
username.add(date) # TODO # correlation message ???
|
||||
|
||||
# if chat: # TODO Chat---Username correlation ???
|
||||
# # Chat---Username
|
||||
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
|
||||
# if meta.get('fwd_from'):
|
||||
# if meta['fwd_from'].get('post_author') # user first name
|
||||
|
||||
# TODO reply threads ????
|
||||
# message edit ????
|
||||
|
||||
return None
|
||||
|
|
|
|||
249
bin/importer/feeders/abstract_chats_feeder.py
Executable file
249
bin/importer/feeders/abstract_chats_feeder.py
Executable file
|
|
@ -0,0 +1,249 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Abstract Chat JSON Feeder Importer Module
|
||||
================
|
||||
|
||||
Process Feeder Json (example: Twitter feeder)
|
||||
|
||||
"""
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
from abc import abstractmethod, ABC
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from importer.feeders.Default import DefaultFeeder
|
||||
from lib.objects.Chats import Chat
|
||||
from lib.objects import ChatSubChannels
|
||||
from lib.objects import Messages
|
||||
from lib.objects import UsersAccount
|
||||
from lib.objects.Usernames import Username
|
||||
from lib import chats_viewer
|
||||
|
||||
import base64
|
||||
import io
|
||||
import gzip
|
||||
|
||||
# TODO remove compression ???
|
||||
def _gunzip_bytes_obj(bytes_obj):
|
||||
gunzipped_bytes_obj = None
|
||||
try:
|
||||
in_ = io.BytesIO()
|
||||
in_.write(bytes_obj)
|
||||
in_.seek(0)
|
||||
|
||||
with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
|
||||
gunzipped_bytes_obj = fo.read()
|
||||
except Exception as e:
|
||||
print(f'Global; Invalid Gzip file: {e}')
|
||||
|
||||
return gunzipped_bytes_obj
|
||||
|
||||
class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||
|
||||
def __init__(self, name, json_data):
|
||||
super().__init__(json_data)
|
||||
self.obj = None
|
||||
self.name = name
|
||||
|
||||
def get_chat_protocol(self): # TODO # # # # # # # # # # # # #
|
||||
return self.name
|
||||
|
||||
def get_chat_network(self):
|
||||
self.json_data['meta'].get('network', None)
|
||||
|
||||
def get_chat_address(self):
|
||||
self.json_data['meta'].get('address', None)
|
||||
|
||||
def get_chat_instance_uuid(self):
|
||||
chat_instance_uuid = chats_viewer.create_chat_service_instance(self.get_chat_protocol(),
|
||||
network=self.get_chat_network(),
|
||||
address=self.get_chat_address())
|
||||
# TODO SET
|
||||
return chat_instance_uuid
|
||||
|
||||
def get_chat_id(self): # TODO RAISE ERROR IF NONE
|
||||
return self.json_data['meta']['chat']['id']
|
||||
|
||||
def get_channel_id(self):
|
||||
pass
|
||||
|
||||
def get_subchannels(self):
|
||||
pass
|
||||
|
||||
def get_thread_id(self):
|
||||
pass
|
||||
|
||||
def get_message_timestamp(self):
|
||||
return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
|
||||
# if self.json_data['meta'].get('date'):
|
||||
# date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
|
||||
# date = date.strftime('%Y/%m/%d')
|
||||
# else:
|
||||
# date = datetime.date.today().strftime("%Y/%m/%d")
|
||||
|
||||
def get_message_date_timestamp(self):
|
||||
timestamp = self.get_message_timestamp()
|
||||
date = datetime.datetime.fromtimestamp(timestamp)
|
||||
date = date.strftime('%Y%m%d')
|
||||
return date, timestamp
|
||||
|
||||
def get_message_sender_id(self):
|
||||
return self.json_data['meta']['sender']['id']
|
||||
|
||||
def get_message_reply(self):
|
||||
return self.json_data['meta'].get('reply_to') # TODO change to reply ???
|
||||
|
||||
def get_message_reply_id(self):
|
||||
return self.json_data['meta'].get('reply_to', None)
|
||||
|
||||
def get_message_content(self):
|
||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||
return _gunzip_bytes_obj(decoded)
|
||||
|
||||
def get_obj(self): # TODO handle others objects -> images, pdf, ...
|
||||
#### TIMESTAMP ####
|
||||
timestamp = self.get_message_timestamp()
|
||||
|
||||
#### Create Object ID ####
|
||||
chat_id = str(self.json_data['meta']['chat']['id'])
|
||||
message_id = str(self.json_data['meta']['id'])
|
||||
# channel id
|
||||
# thread id
|
||||
|
||||
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
|
||||
self.obj = Messages.Message(obj_id)
|
||||
return self.obj
|
||||
|
||||
def process_chat(self, message, date, timestamp, reply_id=None): # TODO threads
|
||||
meta = self.json_data['meta']['chat']
|
||||
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
||||
chat.add(date) # TODO ### Dynamic subtype ???
|
||||
|
||||
if meta.get('name'):
|
||||
chat.set_name(meta['name'])
|
||||
|
||||
if meta.get('date'): # TODO check if already exists
|
||||
chat.set_created_at(int(meta['date']['timestamp']))
|
||||
|
||||
if meta.get('username'):
|
||||
username = Username(meta['username'], self.get_chat_protocol())
|
||||
chat.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
||||
if meta.get('subchannel'):
|
||||
subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id)
|
||||
chat.add_children(obj_global_id=subchannel.get_global_id())
|
||||
else:
|
||||
chat.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
|
||||
|
||||
# if meta.get('subchannels'): # TODO Update icon + names
|
||||
|
||||
return chat
|
||||
|
||||
# def process_subchannels(self):
|
||||
# pass
|
||||
|
||||
def process_subchannel(self, message, date, timestamp, reply_id=None): # TODO CREATE DATE
|
||||
meta = self.json_data['meta']['chat']['subchannel']
|
||||
subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid())
|
||||
subchannel.add(date)
|
||||
|
||||
if meta.get('date'): # TODO check if already exists
|
||||
subchannel.set_created_at(int(meta['date']['timestamp']))
|
||||
|
||||
if meta.get('name'):
|
||||
subchannel.set_name(meta['name'])
|
||||
# subchannel.update_name(meta['name'], timestamp) # TODO #################
|
||||
|
||||
subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
|
||||
return subchannel
|
||||
|
||||
def process_sender(self, date, timestamp):
|
||||
meta = self.json_data['meta']['sender']
|
||||
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
|
||||
|
||||
if meta.get('username'):
|
||||
username = Username(meta['username'], self.get_chat_protocol())
|
||||
# TODO timeline or/and correlation ????
|
||||
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
user_account.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
||||
# Username---Message
|
||||
username.add(date) # TODO # correlation message ???
|
||||
|
||||
# ADDITIONAL METAS
|
||||
if meta.get('firstname'):
|
||||
user_account.set_first_name(meta['firstname'])
|
||||
if meta.get('lastname'):
|
||||
user_account.set_last_name(meta['lastname'])
|
||||
if meta.get('phone'):
|
||||
user_account.set_phone(meta['phone'])
|
||||
|
||||
return user_account
|
||||
|
||||
# Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
|
||||
# 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default
|
||||
# 2) Create/Get CHAT ID - Done
|
||||
# 3) Create/Get Channel IF is in channel
|
||||
# 4) Create/Get Thread IF is in thread
|
||||
# 5) Create/Update Username and User-account - Done
|
||||
def process_meta(self): # TODO CHECK MANDATORY FIELDS
|
||||
"""
|
||||
Process JSON meta filed.
|
||||
"""
|
||||
# meta = self.get_json_meta()
|
||||
|
||||
date, timestamp = self.get_message_date_timestamp()
|
||||
|
||||
# REPLY
|
||||
reply_id = self.get_message_reply_id()
|
||||
|
||||
# TODO Translation
|
||||
|
||||
# Content
|
||||
content = self.get_message_content()
|
||||
|
||||
message = Messages.create(self.obj.id, content) # TODO translation
|
||||
|
||||
# CHAT
|
||||
chat = self.process_chat(message, date, timestamp, reply_id=reply_id)
|
||||
|
||||
# SENDER # TODO HANDLE NULL SENDER
|
||||
user_account = self.process_sender(date, timestamp)
|
||||
|
||||
# UserAccount---Message
|
||||
user_account.add(date, obj=message)
|
||||
# UserAccount---Chat
|
||||
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
|
||||
|
||||
# if chat: # TODO Chat---Username correlation ???
|
||||
# # Chat---Username
|
||||
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -13,6 +13,7 @@ from lib.ConfigLoader import ConfigLoader
|
|||
|
||||
config_loader = ConfigLoader()
|
||||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
|
||||
|
|
@ -40,9 +41,11 @@ def get_all_objects():
|
|||
def get_objects_with_subtypes():
|
||||
return ['chat', 'cryptocurrency', 'pgp', 'username']
|
||||
|
||||
def get_object_all_subtypes(obj_type):
|
||||
def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
|
||||
if obj_type == 'chat':
|
||||
return ['discord', 'jabber', 'telegram']
|
||||
return r_object.smembers(f'all_chat:subtypes')
|
||||
if obj_type == 'chat-subchannel':
|
||||
return r_object.smembers(f'all_chat-subchannel:subtypes')
|
||||
if obj_type == 'cryptocurrency':
|
||||
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
|
||||
if obj_type == 'pgp':
|
||||
|
|
|
|||
305
bin/lib/chats_viewer.py
Executable file
305
bin/lib/chats_viewer.py
Executable file
|
|
@ -0,0 +1,305 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
Chats Viewer
|
||||
===================
|
||||
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects import Chats
|
||||
from lib.objects import ChatSubChannels
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
r_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
r_crawler = config_loader.get_db_conn("Kvrocks_Crawler")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
|
||||
r_obj = config_loader.get_db_conn("Kvrocks_DB") # TEMP new DB ????
|
||||
|
||||
# # # # # # # #
|
||||
# #
|
||||
# COMMON #
|
||||
# #
|
||||
# # # # # # # #
|
||||
|
||||
# TODO ChatDefaultPlatform
|
||||
|
||||
# CHAT(type=chat, subtype=platform, id= chat_id)
|
||||
|
||||
# Channel(type=channel, subtype=platform, id=channel_id)
|
||||
|
||||
# Thread(type=thread, subtype=platform, id=thread_id)
|
||||
|
||||
# Message(type=message, subtype=platform, id=message_id)
|
||||
|
||||
|
||||
# Protocol/Platform
|
||||
|
||||
|
||||
# class ChatProtocols: # TODO Remove Me
|
||||
#
|
||||
# def __init__(self): # name ???? subtype, id ????
|
||||
# # discord, mattermost, ...
|
||||
# pass
|
||||
#
|
||||
# def get_chat_protocols(self):
|
||||
# pass
|
||||
#
|
||||
# def get_chat_protocol(self, protocol):
|
||||
# pass
|
||||
#
|
||||
# ################################################################
|
||||
#
|
||||
# def get_instances(self):
|
||||
# pass
|
||||
#
|
||||
# def get_chats(self):
|
||||
# pass
|
||||
#
|
||||
# def get_chats_by_instance(self, instance):
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# class ChatNetwork: # uuid or protocol
|
||||
# def __init__(self, network='default'):
|
||||
# self.id = network
|
||||
#
|
||||
# def get_addresses(self):
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# class ChatServerAddress: # uuid or protocol + network
|
||||
# def __init__(self, address='default'):
|
||||
# self.id = address
|
||||
|
||||
# map uuid -> type + field
|
||||
|
||||
# TODO option last protocol/ imported messages/chat -> unread mode ????
|
||||
|
||||
# # # # # # # # #
|
||||
# #
|
||||
# PROTOCOLS # IRC, discord, mattermost, ...
|
||||
# #
|
||||
# # # # # # # # # TODO icon => UI explorer by protocol + network + instance
|
||||
|
||||
def get_chat_protocols():
|
||||
return r_obj.smembers(f'chat:protocols')
|
||||
|
||||
def get_chat_protocols_meta():
|
||||
metas = []
|
||||
for protocol_id in get_chat_protocols():
|
||||
protocol = ChatProtocol(protocol_id)
|
||||
metas.append(protocol.get_meta(options={'icon'}))
|
||||
return metas
|
||||
|
||||
class ChatProtocol: # TODO first seen last seen ???? + nb by day ????
|
||||
def __init__(self, protocol):
|
||||
self.id = protocol
|
||||
|
||||
def exists(self):
|
||||
return r_db.exists(f'chat:protocol:{self.id}')
|
||||
|
||||
def get_networks(self):
|
||||
return r_db.smembers(f'chat:protocol:{self.id}')
|
||||
|
||||
def get_nb_networks(self):
|
||||
return r_db.scard(f'chat:protocol:{self.id}')
|
||||
|
||||
def get_icon(self):
|
||||
if self.id == 'discord':
|
||||
icon = {'style': 'fab', 'icon': 'fa-discord'}
|
||||
elif self.id == 'telegram':
|
||||
icon = {'style': 'fab', 'icon': 'fa-telegram'}
|
||||
else:
|
||||
icon = {}
|
||||
return icon
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = {'id': self.id}
|
||||
if 'icon' in options:
|
||||
meta['icon'] = self.get_icon()
|
||||
return meta
|
||||
|
||||
# def get_addresses(self):
|
||||
# pass
|
||||
#
|
||||
# def get_instances_uuids(self):
|
||||
# pass
|
||||
|
||||
|
||||
# # # # # # # # # # # # # #
|
||||
# #
|
||||
# ChatServiceInstance #
|
||||
# #
|
||||
# # # # # # # # # # # # # #
|
||||
|
||||
# uuid -> protocol + network + server
|
||||
class ChatServiceInstance:
|
||||
def __init__(self, instance_uuid):
|
||||
self.uuid = instance_uuid
|
||||
|
||||
def exists(self):
|
||||
return r_obj.exists(f'chatSerIns:{self.uuid}')
|
||||
|
||||
def get_protocol(self): # return objects ????
|
||||
return r_obj.hget(f'chatSerIns:{self.uuid}', 'protocol')
|
||||
|
||||
def get_network(self): # return objects ????
|
||||
network = r_obj.hget(f'chatSerIns:{self.uuid}', 'network')
|
||||
if network:
|
||||
return network
|
||||
|
||||
def get_address(self): # return objects ????
|
||||
address = r_obj.hget(f'chatSerIns:{self.uuid}', 'address')
|
||||
if address:
|
||||
return address
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = {'uuid': self.uuid,
|
||||
'protocol': self.get_protocol(),
|
||||
'network': self.get_network(),
|
||||
'address': self.get_address()}
|
||||
if 'chats' in options:
|
||||
meta['chats'] = []
|
||||
for chat_id in self.get_chats():
|
||||
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'}))
|
||||
return meta
|
||||
|
||||
def get_nb_chats(self):
|
||||
return Chats.Chats().get_nb_ids_by_subtype(self.uuid)
|
||||
|
||||
def get_chats(self):
|
||||
return Chats.Chats().get_ids_by_subtype(self.uuid)
|
||||
|
||||
def get_chat_service_instances():
|
||||
return r_obj.smembers(f'chatSerIns:all')
|
||||
|
||||
def get_chat_service_instance_uuid(protocol, network, address):
|
||||
if not network:
|
||||
network = ''
|
||||
if not address:
|
||||
address = ''
|
||||
return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address)
|
||||
|
||||
def get_chat_service_instance(protocol, network, address):
|
||||
instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
|
||||
if instance_uuid:
|
||||
return ChatServiceInstance(instance_uuid)
|
||||
|
||||
def create_chat_service_instance(protocol, network=None, address=None):
|
||||
instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
|
||||
if instance_uuid:
|
||||
return instance_uuid
|
||||
else:
|
||||
if not network:
|
||||
network = ''
|
||||
if not address:
|
||||
address = ''
|
||||
instance_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f'{protocol}|{network}|{address}'))
|
||||
r_obj.sadd(f'chatSerIns:all', instance_uuid)
|
||||
|
||||
# map instance - uuid
|
||||
r_obj.hset(f'map:chatSerIns:{protocol}:{network}', address, instance_uuid)
|
||||
|
||||
r_obj.hset(f'chatSerIns:{instance_uuid}', 'protocol', protocol)
|
||||
if network:
|
||||
r_obj.hset(f'chatSerIns:{instance_uuid}', 'network', network)
|
||||
if address:
|
||||
r_obj.hset(f'chatSerIns:{instance_uuid}', 'address', address)
|
||||
|
||||
# protocols
|
||||
r_obj.sadd(f'chat:protocols', protocol) # TODO first seen / last seen
|
||||
|
||||
# protocol -> network
|
||||
r_obj.sadd(f'chat:protocol:networks:{protocol}', network)
|
||||
|
||||
return instance_uuid
|
||||
|
||||
|
||||
|
||||
|
||||
# INSTANCE ===> CHAT IDS
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# protocol -> instance_uuids => for protocol->networks -> protocol+network => HGETALL
|
||||
# protocol+network -> instance_uuids => HGETALL
|
||||
|
||||
# protocol -> networks ???default??? or ''
|
||||
|
||||
# --------------------------------------------------------
|
||||
# protocol+network -> addresses => HKEYS
|
||||
# protocol+network+addresse => HGET
|
||||
|
||||
|
||||
# Chat -> subtype=uuid, id = chat id
|
||||
|
||||
|
||||
# instance_uuid -> chat id
|
||||
|
||||
|
||||
# protocol - uniq ID
|
||||
# protocol + network -> uuid ????
|
||||
# protocol + network + address -> uuid
|
||||
|
||||
#######################################################################################
|
||||
|
||||
def get_subchannels_meta_from_global_id(subchannels):
|
||||
meta = []
|
||||
for sub in subchannels:
|
||||
_, instance_uuid, sub_id = sub.split(':', 2)
|
||||
subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
|
||||
meta.append(subchannel.get_meta({'nb_messages'}))
|
||||
return meta
|
||||
|
||||
def api_get_chat_service_instance(chat_instance_uuid):
|
||||
chat_instance = ChatServiceInstance(chat_instance_uuid)
|
||||
if not chat_instance.exists():
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
return chat_instance.get_meta({'chats'}), 200
|
||||
|
||||
def api_get_chat(chat_id, chat_instance_uuid):
|
||||
chat = Chats.Chat(chat_id, chat_instance_uuid)
|
||||
if not chat.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
meta = chat.get_meta({'img', 'subchannels', 'username'})
|
||||
if meta['subchannels']:
|
||||
print(meta['subchannels'])
|
||||
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
|
||||
return meta, 200
|
||||
|
||||
def api_get_subchannel(chat_id, chat_instance_uuid):
|
||||
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
|
||||
if not subchannel.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
meta = subchannel.get_meta({'img', 'nb_messages'})
|
||||
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
|
||||
return meta, 200
|
||||
|
||||
# # # # # # # # # # LATER
|
||||
# #
|
||||
# ChatCategory #
|
||||
# #
|
||||
# # # # # # # # # #
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
r = get_chat_service_instances()
|
||||
print(r)
|
||||
r = ChatServiceInstance(r.pop())
|
||||
print(r.get_meta({'chats'}))
|
||||
# r = get_chat_protocols()
|
||||
# print(r)
|
||||
153
bin/lib/objects/ChatSubChannels.py
Executable file
153
bin/lib/objects/ChatSubChannels.py
Executable file
|
|
@ -0,0 +1,153 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from flask import url_for
|
||||
# from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ail_core
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
|
||||
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib.objects import ail_objects
|
||||
from lib.timeline_engine import Timeline
|
||||
|
||||
from lib.correlations_engine import get_correlation_by_correl_type
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
config_loader = None
|
||||
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
||||
class ChatSubChannel(AbstractChatObject):
|
||||
"""
|
||||
AIL Chat Object. (strings)
|
||||
"""
|
||||
|
||||
# ID -> <CHAT ID>/<SubChannel ID> subtype = chat_instance_uuid
|
||||
def __init__(self, id, subtype):
|
||||
super(ChatSubChannel, self).__init__('chat-subchannel', id, subtype)
|
||||
|
||||
# def get_ail_2_ail_payload(self):
|
||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||
# 'compress': 'gzip'}
|
||||
# return payload
|
||||
|
||||
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||
def delete(self):
|
||||
# # TODO:
|
||||
pass
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
|
||||
else:
|
||||
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
|
||||
return url
|
||||
|
||||
def get_svg_icon(self): # TODO
|
||||
# if self.subtype == 'telegram':
|
||||
# style = 'fab'
|
||||
# icon = '\uf2c6'
|
||||
# elif self.subtype == 'discord':
|
||||
# style = 'fab'
|
||||
# icon = '\uf099'
|
||||
# else:
|
||||
# style = 'fas'
|
||||
# icon = '\uf007'
|
||||
style = 'fas'
|
||||
icon = '\uf086'
|
||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
||||
|
||||
# TODO TIME LAST MESSAGES
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
meta['name'] = self.get_name()
|
||||
if 'img' in options:
|
||||
meta['sub'] = self.get_img()
|
||||
if 'nb_messages':
|
||||
meta['nb_messages'] = self.get_nb_messages()
|
||||
return meta
|
||||
|
||||
def get_misp_object(self):
|
||||
# obj_attrs = []
|
||||
# if self.subtype == 'telegram':
|
||||
# obj = MISPObject('telegram-account', standalone=True)
|
||||
# obj_attrs.append(obj.add_attribute('username', value=self.id))
|
||||
#
|
||||
# elif self.subtype == 'twitter':
|
||||
# obj = MISPObject('twitter-account', standalone=True)
|
||||
# obj_attrs.append(obj.add_attribute('name', value=self.id))
|
||||
#
|
||||
# else:
|
||||
# obj = MISPObject('user-account', standalone=True)
|
||||
# obj_attrs.append(obj.add_attribute('username', value=self.id))
|
||||
#
|
||||
# first_seen = self.get_first_seen()
|
||||
# last_seen = self.get_last_seen()
|
||||
# if first_seen:
|
||||
# obj.first_seen = first_seen
|
||||
# if last_seen:
|
||||
# obj.last_seen = last_seen
|
||||
# if not first_seen or not last_seen:
|
||||
# self.logger.warning(
|
||||
# f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
|
||||
#
|
||||
# for obj_attr in obj_attrs:
|
||||
# for tag in self.get_tags():
|
||||
# obj_attr.add_tag(tag)
|
||||
# return obj
|
||||
return
|
||||
|
||||
############################################################################
|
||||
############################################################################
|
||||
|
||||
# others optional metas, ... -> # TODO ALL meta in hset
|
||||
|
||||
def _get_timeline_name(self):
|
||||
return Timeline(self.get_global_id(), 'username')
|
||||
|
||||
def update_name(self, name, timestamp):
|
||||
self._get_timeline_name().add_timestamp(timestamp, name)
|
||||
|
||||
|
||||
# TODO # # # # # # # # # # #
|
||||
def get_users(self):
|
||||
pass
|
||||
|
||||
#### Categories ####
|
||||
|
||||
#### Threads ####
|
||||
|
||||
#### Messages #### TODO set parents
|
||||
|
||||
# def get_last_message_id(self):
|
||||
#
|
||||
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
|
||||
|
||||
|
||||
class ChatSubChannels(AbstractChatObjects):
|
||||
def __init__(self):
|
||||
super().__init__('chat-subchannels')
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# chat = Chat('test', 'telegram')
|
||||
# r = chat.get_messages()
|
||||
# print(r)
|
||||
103
bin/lib/objects/ChatThreads.py
Executable file
103
bin/lib/objects/ChatThreads.py
Executable file
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from flask import url_for
|
||||
# from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ail_core
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib.objects import ail_objects
|
||||
from lib.timeline_engine import Timeline
|
||||
|
||||
from lib.correlations_engine import get_correlation_by_correl_type
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
config_loader = None
|
||||
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
||||
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
||||
"""
|
||||
AIL Chat Object. (strings)
|
||||
"""
|
||||
|
||||
def __init__(self, id, subtype):
|
||||
super(Chat, self).__init__('chat-thread', id, subtype)
|
||||
|
||||
# def get_ail_2_ail_payload(self):
|
||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||
# 'compress': 'gzip'}
|
||||
# return payload
|
||||
|
||||
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||
def delete(self):
|
||||
# # TODO:
|
||||
pass
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
|
||||
else:
|
||||
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
|
||||
return url
|
||||
|
||||
def get_svg_icon(self): # TODO
|
||||
# if self.subtype == 'telegram':
|
||||
# style = 'fab'
|
||||
# icon = '\uf2c6'
|
||||
# elif self.subtype == 'discord':
|
||||
# style = 'fab'
|
||||
# icon = '\uf099'
|
||||
# else:
|
||||
# style = 'fas'
|
||||
# icon = '\uf007'
|
||||
style = 'fas'
|
||||
icon = '\uf086'
|
||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['subtype'] = self.subtype
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
if 'username':
|
||||
meta['username'] = self.get_username()
|
||||
return meta
|
||||
|
||||
def get_misp_object(self):
|
||||
return
|
||||
|
||||
############################################################################
|
||||
############################################################################
|
||||
|
||||
# others optional metas, ... -> # TODO ALL meta in hset
|
||||
|
||||
#### Messages #### TODO set parents
|
||||
|
||||
# def get_last_message_id(self):
|
||||
#
|
||||
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chat = Chat('test', 'telegram')
|
||||
r = chat.get_messages()
|
||||
print(r)
|
||||
|
|
@ -15,6 +15,9 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from lib import ail_core
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
|
||||
|
||||
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib.objects import ail_objects
|
||||
|
|
@ -33,19 +36,14 @@ config_loader = None
|
|||
################################################################################
|
||||
################################################################################
|
||||
|
||||
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
||||
class Chat(AbstractChatObject):
|
||||
"""
|
||||
AIL Chat Object. (strings)
|
||||
AIL Chat Object.
|
||||
"""
|
||||
|
||||
def __init__(self, id, subtype):
|
||||
super(Chat, self).__init__('chat', id, subtype)
|
||||
|
||||
# def get_ail_2_ail_payload(self):
|
||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||
# 'compress': 'gzip'}
|
||||
# return payload
|
||||
|
||||
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||
def delete(self):
|
||||
# # TODO:
|
||||
|
|
@ -74,9 +72,16 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['subtype'] = self.subtype
|
||||
meta['name'] = self.get_name()
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
if 'img':
|
||||
meta['icon'] = self.get_img()
|
||||
if 'username' in options:
|
||||
meta['username'] = self.get_username()
|
||||
if 'subchannels' in options:
|
||||
meta['subchannels'] = self.get_subchannels()
|
||||
if 'nb_subchannels':
|
||||
meta['nb_subchannels'] = self.get_nb_subchannels()
|
||||
return meta
|
||||
|
||||
def get_misp_object(self):
|
||||
|
|
@ -112,11 +117,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
############################################################################
|
||||
############################################################################
|
||||
|
||||
# others optional metas, ... -> # TODO ALL meta in hset
|
||||
|
||||
def get_name(self): # get username ????
|
||||
pass
|
||||
|
||||
# users that send at least a message else participants/spectator
|
||||
# correlation created by messages
|
||||
def get_users(self):
|
||||
|
|
@ -138,22 +138,22 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
def update_username_timeline(self, username_global_id, timestamp):
|
||||
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
|
||||
|
||||
#### ChatSubChannels ####
|
||||
|
||||
|
||||
#### Categories ####
|
||||
|
||||
#### Threads ####
|
||||
|
||||
#### Messages #### TODO set parents
|
||||
|
||||
# def get_last_message_id(self):
|
||||
#
|
||||
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
|
||||
|
||||
def get_obj_message_id(self, obj_id):
|
||||
if obj_id.endswith('.gz'):
|
||||
obj_id = obj_id[:-3]
|
||||
return int(obj_id.split('_')[-1])
|
||||
|
||||
def _get_message_timestamp(self, obj_global_id):
|
||||
return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
|
||||
|
||||
def _get_messages(self):
|
||||
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
|
||||
|
||||
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
|
||||
obj = ail_objects.get_obj_from_global_id(obj_global_id)
|
||||
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
|
||||
|
|
@ -179,26 +179,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
|
||||
return mess_dict
|
||||
|
||||
|
||||
def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies
|
||||
start = 0
|
||||
stop = -1
|
||||
# r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}')
|
||||
|
||||
# TODO chat without username ???? -> chat ID ????
|
||||
|
||||
messages = {}
|
||||
curr_date = None
|
||||
for message in self._get_messages():
|
||||
date = datetime.fromtimestamp(message[1])
|
||||
date_day = date.strftime('%Y/%m/%d')
|
||||
if date_day != curr_date:
|
||||
messages[date_day] = []
|
||||
curr_date = date_day
|
||||
mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)
|
||||
messages[date_day].append(mess_dict)
|
||||
return messages
|
||||
|
||||
# Zset with ID ??? id -> item id ??? multiple id == media + text
|
||||
# id -> media id
|
||||
# How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
|
||||
|
|
@ -229,43 +209,12 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
|
|||
# domain = get_item_domain(item_id)
|
||||
# self.add_correlation('domain', '', domain)
|
||||
|
||||
# TODO kvrocks exception if key don't exists
|
||||
def get_obj_by_message_id(self, mess_id):
|
||||
return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id)
|
||||
|
||||
# importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn
|
||||
def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None):
|
||||
r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id)
|
||||
r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp})
|
||||
|
||||
if reply_id:
|
||||
reply_obj = self.get_obj_by_message_id(reply_id)
|
||||
if reply_obj:
|
||||
self.add_obj_children(reply_obj, obj_global_id)
|
||||
else:
|
||||
self.add_message_cached_reply(reply_id, mess_id)
|
||||
|
||||
# ADD cached replies
|
||||
for reply_obj in self.get_cached_message_reply(mess_id):
|
||||
self.add_obj_children(obj_global_id, reply_obj)
|
||||
|
||||
def _get_message_cached_reply(self, message_id):
|
||||
return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
|
||||
|
||||
def get_cached_message_reply(self, message_id):
|
||||
objs_global_id = []
|
||||
for mess_id in self._get_message_cached_reply(message_id):
|
||||
obj_global_id = self.get_obj_by_message_id(mess_id)
|
||||
if obj_global_id:
|
||||
objs_global_id.append(obj_global_id)
|
||||
return objs_global_id
|
||||
|
||||
def add_message_cached_reply(self, reply_to_id, message_id):
|
||||
r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id)
|
||||
r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600)
|
||||
|
||||
# TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering
|
||||
|
||||
class Chats(AbstractChatObjects):
|
||||
def __init__(self):
|
||||
super().__init__('chat')
|
||||
|
||||
# TODO factorize
|
||||
def get_all_subtypes():
|
||||
|
|
@ -280,28 +229,6 @@ def get_all():
|
|||
def get_all_by_subtype(subtype):
|
||||
return get_all_id('chat', subtype)
|
||||
|
||||
# # TODO FILTER NAME + Key + mail
|
||||
# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
|
||||
#
|
||||
# return name_to_search
|
||||
#
|
||||
# def search_usernames_by_name(name_to_search, subtype, r_pos=False):
|
||||
# usernames = {}
|
||||
# # for subtype in subtypes:
|
||||
# r_name = sanitize_username_name_to_search(name_to_search, subtype)
|
||||
# if not name_to_search or isinstance(r_name, dict):
|
||||
# # break
|
||||
# return usernames
|
||||
# r_name = re.compile(r_name)
|
||||
# for user_name in get_all_usernames_by_subtype(subtype):
|
||||
# res = re.search(r_name, user_name)
|
||||
# if res:
|
||||
# usernames[user_name] = {}
|
||||
# if r_pos:
|
||||
# usernames[user_name]['hl-start'] = res.start()
|
||||
# usernames[user_name]['hl-end'] = res.end()
|
||||
# return usernames
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chat = Chat('test', 'telegram')
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ config_loader = None
|
|||
# /!\ handle null chat and message id -> chat = uuid and message = timestamp ???
|
||||
|
||||
|
||||
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<message ID> => telegram without channels
|
||||
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<message ID>
|
||||
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Thread ID>/<message ID>
|
||||
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<Thread ID>/<message ID>
|
||||
class Message(AbstractObject):
|
||||
"""
|
||||
AIL Message Object. (strings)
|
||||
|
|
@ -86,7 +90,7 @@ class Message(AbstractObject):
|
|||
return dirs[-2]
|
||||
|
||||
def get_message_id(self): # TODO optimize
|
||||
message_id = self.get_basename().rsplit('_', 1)[1]
|
||||
message_id = self.get_basename().rsplit('/', 1)[1]
|
||||
# if message_id.endswith('.gz'):
|
||||
# message_id = message_id[:-3]
|
||||
return message_id
|
||||
|
|
@ -97,6 +101,10 @@ class Message(AbstractObject):
|
|||
# chat_id = chat_id[:-3]
|
||||
return chat_id
|
||||
|
||||
# TODO get Instance ID
|
||||
# TODO get channel ID
|
||||
# TODO get thread ID
|
||||
|
||||
def get_user_account(self):
|
||||
user_account = self.get_correlation('user-account')
|
||||
if user_account.get('user-account'):
|
||||
|
|
@ -255,8 +263,16 @@ class Message(AbstractObject):
|
|||
def delete(self):
|
||||
pass
|
||||
|
||||
def create_obj_id(source, chat_id, message_id, timestamp):
|
||||
return f'{source}/{timestamp}/{chat_id}_{message_id}'
|
||||
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None):
|
||||
timestamp = int(timestamp)
|
||||
if channel_id and thread_id:
|
||||
return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ?????
|
||||
elif channel_id:
|
||||
return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
|
||||
elif thread_id:
|
||||
return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}'
|
||||
else:
|
||||
return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
|
||||
|
||||
# TODO Check if already exists
|
||||
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
|
||||
|
|
|
|||
241
bin/lib/objects/abstract_chat_object.py
Executable file
241
bin/lib/objects/abstract_chat_object.py
Executable file
|
|
@ -0,0 +1,241 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Base Class for AIL Objects
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import sys
|
||||
from abc import ABC
|
||||
|
||||
from datetime import datetime
|
||||
# from flask import url_for
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject
|
||||
from lib.ail_core import get_object_all_subtypes, zscan_iter ################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Messages import Message
|
||||
from lib.objects.UsersAccount import UserAccount
|
||||
from lib.objects.Usernames import Username
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
|
||||
from packages import Date
|
||||
|
||||
# LOAD CONFIG
|
||||
config_loader = ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
# # FIXME: SAVE SUBTYPE NAMES ?????
|
||||
|
||||
class AbstractChatObject(AbstractSubtypeObject, ABC):
|
||||
"""
|
||||
Abstract Subtype Object
|
||||
"""
|
||||
|
||||
def __init__(self, obj_type, id, subtype):
|
||||
""" Abstract for all the AIL object
|
||||
|
||||
:param obj_type: object type (item, ...)
|
||||
:param id: Object ID
|
||||
"""
|
||||
super().__init__(obj_type, id, subtype)
|
||||
|
||||
# get useraccount / username
|
||||
# get users ?
|
||||
# timeline name ????
|
||||
# info
|
||||
# created
|
||||
# last imported/updated
|
||||
|
||||
# TODO get instance
|
||||
# TODO get protocol
|
||||
# TODO get network
|
||||
# TODO get address
|
||||
|
||||
def get_chat(self): # require ail object TODO ##
|
||||
if self.type != 'chat':
|
||||
parent = self.get_parent()
|
||||
obj_type, _ = parent.split(':', 1)
|
||||
if obj_type == 'chat':
|
||||
return parent
|
||||
|
||||
def get_subchannels(self):
|
||||
subchannels = []
|
||||
if self.type == 'chat': # category ???
|
||||
print(self.get_childrens())
|
||||
for obj_global_id in self.get_childrens():
|
||||
print(obj_global_id)
|
||||
obj_type, _ = obj_global_id.split(':', 1)
|
||||
if obj_type == 'chat-subchannel':
|
||||
subchannels.append(obj_global_id)
|
||||
return subchannels
|
||||
|
||||
def get_nb_subchannels(self):
|
||||
nb = 0
|
||||
if self.type == 'chat':
|
||||
for obj_global_id in self.get_childrens():
|
||||
obj_type, _ = obj_global_id.split(':', 1)
|
||||
if obj_type == 'chat-subchannel':
|
||||
nb += 1
|
||||
return nb
|
||||
|
||||
def get_threads(self):
|
||||
threads = []
|
||||
for obj_global_id in self.get_childrens():
|
||||
obj_type, _ = obj_global_id.split(':', 1)
|
||||
if obj_type == 'chat-thread':
|
||||
threads.append(obj_global_id)
|
||||
return threads
|
||||
|
||||
def get_created_at(self):
|
||||
return self._get_field('created_at')
|
||||
|
||||
def set_created_at(self, timestamp):
|
||||
self._set_field('created_at', timestamp)
|
||||
|
||||
def get_name(self):
|
||||
name = self._get_field('name')
|
||||
if not name:
|
||||
name = ''
|
||||
return name
|
||||
|
||||
def set_name(self, name):
|
||||
self._set_field('name', name)
|
||||
|
||||
def get_img(self):
|
||||
return self._get_field('img')
|
||||
|
||||
def set_img(self, icon):
|
||||
self._set_field('img', icon)
|
||||
|
||||
def get_nb_messages(self):
|
||||
return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
|
||||
|
||||
def _get_messages(self): # TODO paginate
|
||||
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
|
||||
|
||||
def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message
|
||||
obj = Message(message[9:])
|
||||
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
|
||||
print(mess_dict)
|
||||
if mess_dict.get('parent') and parent:
|
||||
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
|
||||
if mess_dict.get('user-account'):
|
||||
_, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3)
|
||||
print(mess_dict['user-account'])
|
||||
user_account = UserAccount(user_account_id, user_account_subtype)
|
||||
mess_dict['user-account'] = {}
|
||||
mess_dict['user-account']['type'] = user_account.get_type()
|
||||
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
|
||||
mess_dict['user-account']['id'] = user_account.get_id()
|
||||
username = user_account.get_username()
|
||||
if username:
|
||||
_, username_account_subtype, username_account_id = username.split(':', 3)
|
||||
username = Username(username_account_id, username_account_subtype).get_default_meta(link=False)
|
||||
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
|
||||
else:
|
||||
mess_dict['user-account'] = {'id': 'UNKNOWN'}
|
||||
|
||||
if not mess_datetime:
|
||||
obj_mess_id = message.get_timestamp()
|
||||
mess_datetime = datetime.fromtimestamp(obj_mess_id)
|
||||
mess_dict['date'] = mess_datetime.isoformat(' ')
|
||||
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
|
||||
return mess_dict
|
||||
|
||||
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ????
|
||||
# TODO return message meta
|
||||
tags = {}
|
||||
messages = {}
|
||||
curr_date = None
|
||||
for message in self._get_messages():
|
||||
date = datetime.fromtimestamp(message[1])
|
||||
date_day = date.strftime('%Y/%m/%d')
|
||||
if date_day != curr_date:
|
||||
messages[date_day] = []
|
||||
curr_date = date_day
|
||||
mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object
|
||||
messages[date_day].append(mess_dict)
|
||||
|
||||
if mess_dict.get('tags'):
|
||||
for tag in mess_dict['tags']:
|
||||
if tag not in tags:
|
||||
tags[tag] = 0
|
||||
tags[tag] += 1
|
||||
return messages, tags
|
||||
|
||||
# TODO REWRITE ADD OR ADD MESSAGE ????
|
||||
# add
|
||||
# add message
|
||||
|
||||
def get_obj_by_message_id(self, message_id):
|
||||
return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id)
|
||||
|
||||
def add_message_cached_reply(self, reply_id, message_id):
|
||||
r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id)
|
||||
r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600)
|
||||
|
||||
def _get_message_cached_reply(self, message_id):
|
||||
return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
|
||||
|
||||
def get_cached_message_reply(self, message_id):
|
||||
objs_global_id = []
|
||||
for mess_id in self._get_message_cached_reply(message_id):
|
||||
obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION
|
||||
if obj_global_id:
|
||||
objs_global_id.append(obj_global_id)
|
||||
return objs_global_id
|
||||
|
||||
def add_message(self, obj_global_id, message_id, timestamp, reply_id=None):
|
||||
r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id)
|
||||
r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)})
|
||||
|
||||
# MESSAGE REPLY
|
||||
if reply_id:
|
||||
reply_obj = self.get_obj_by_message_id(reply_id) # TODO CATCH EXCEPTION
|
||||
if reply_obj:
|
||||
self.add_obj_children(reply_obj, obj_global_id)
|
||||
else:
|
||||
self.add_message_cached_reply(reply_id, message_id)
|
||||
|
||||
|
||||
# get_messages_meta ????
|
||||
|
||||
# TODO move me to abstract subtype
|
||||
class AbstractChatObjects(ABC):
|
||||
def __init__(self, type):
|
||||
self.type = type
|
||||
|
||||
def add_subtype(self, subtype):
|
||||
r_object.sadd(f'all_{self.type}:subtypes', subtype)
|
||||
|
||||
def get_subtypes(self):
|
||||
return r_object.smembers(f'all_{self.type}:subtypes')
|
||||
|
||||
def get_nb_ids_by_subtype(self, subtype):
|
||||
return r_object.zcard(f'{self.type}_all:{subtype}')
|
||||
|
||||
def get_ids_by_subtype(self, subtype):
|
||||
print(subtype)
|
||||
print(f'{self.type}_all:{subtype}')
|
||||
return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
|
||||
|
||||
def get_all_id_iterator_iter(self, subtype):
|
||||
return zscan_iter(r_object, f'{self.type}_all:{subtype}')
|
||||
|
||||
def get_ids(self):
|
||||
pass
|
||||
|
||||
def search(self):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
|
@ -289,23 +289,24 @@ class AbstractObject(ABC):
|
|||
def get_parent(self):
|
||||
return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent')
|
||||
|
||||
def get_children(self):
|
||||
def get_childrens(self):
|
||||
return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
|
||||
|
||||
def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
|
||||
def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP
|
||||
if not obj_global_id:
|
||||
if obj_subtype is None:
|
||||
obj_subtype = ''
|
||||
obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
|
||||
r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id)
|
||||
|
||||
def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
|
||||
def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP
|
||||
if not obj_global_id:
|
||||
if obj_subtype is None:
|
||||
obj_subtype = ''
|
||||
obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
|
||||
r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id)
|
||||
|
||||
## others objects ##
|
||||
def add_obj_children(self, parent_global_id, son_global_id):
|
||||
r_object.sadd(f'child:{parent_global_id}', son_global_id)
|
||||
r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id)
|
||||
|
|
|
|||
|
|
@ -88,7 +88,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
def _get_meta(self, options=None):
|
||||
if options is None:
|
||||
options = set()
|
||||
meta = {'first_seen': self.get_first_seen(),
|
||||
meta = {'id': self.id,
|
||||
'subtype': self.subtype,
|
||||
'first_seen': self.get_first_seen(),
|
||||
'last_seen': self.get_last_seen(),
|
||||
'nb_seen': self.get_nb_seen()}
|
||||
if 'icon' in options:
|
||||
|
|
@ -150,8 +152,11 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
# => data Retention + efficient search
|
||||
#
|
||||
#
|
||||
def _add_subtype(self):
|
||||
r_object.sadd(f'all_{self.type}:subtypes', self.subtype)
|
||||
|
||||
def add(self, date, obj=None):
|
||||
self._add_subtype()
|
||||
self.update_daterange(date)
|
||||
update_obj_date(date, self.type, self.subtype)
|
||||
# daily
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ from blueprints.objects_title import objects_title
|
|||
from blueprints.objects_cookie_name import objects_cookie_name
|
||||
from blueprints.objects_etag import objects_etag
|
||||
from blueprints.objects_hhhash import objects_hhhash
|
||||
from blueprints.objects_chat import objects_chat
|
||||
from blueprints.chats_explorer import chats_explorer
|
||||
|
||||
Flask_dir = os.environ['AIL_FLASK']
|
||||
|
||||
|
|
@ -108,7 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl)
|
|||
app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_etag, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_chat, url_prefix=baseUrl)
|
||||
app.register_blueprint(chats_explorer, url_prefix=baseUrl)
|
||||
|
||||
# ========= =========#
|
||||
|
||||
|
|
|
|||
132
var/www/blueprints/chats_explorer.py
Normal file
132
var/www/blueprints/chats_explorer.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import chats_viewer
|
||||
|
||||
|
||||
|
||||
############################################
|
||||
|
||||
from lib import ail_core
|
||||
from lib.objects import ail_objects
|
||||
from lib import chats_viewer
|
||||
from lib.objects import Chats
|
||||
from lib.objects import ChatSubChannels
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
def create_json_response(data, status_code):
|
||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@chats_explorer.route("/chats/explorer", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def chats_explorer_dashboard():
|
||||
return
|
||||
|
||||
@chats_explorer.route("chats/explorer/protocols", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def chats_explorer_protocols():
|
||||
protocols = chats_viewer.get_chat_protocols_meta()
|
||||
return render_template('chats_protocols.html', protocols=protocols)
|
||||
|
||||
@chats_explorer.route("chats/explorer/instance", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def chats_explorer_instance():
|
||||
intance_uuid = request.args.get('uuid')
|
||||
chat_instance = chats_viewer.api_get_chat_service_instance(intance_uuid)
|
||||
if chat_instance[1] != 200:
|
||||
return create_json_response(chat_instance[0], chat_instance[1])
|
||||
else:
|
||||
chat_instance = chat_instance[0]
|
||||
return render_template('chat_instance.html', chat_instance=chat_instance)
|
||||
|
||||
@chats_explorer.route("chats/explorer/chat", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def chats_explorer_chat():
|
||||
chat_id = request.args.get('id')
|
||||
instance_uuid = request.args.get('uuid')
|
||||
chat = chats_viewer.api_get_chat(chat_id, instance_uuid)
|
||||
if chat[1] != 200:
|
||||
return create_json_response(chat[0], chat[1])
|
||||
else:
|
||||
chat = chat[0]
|
||||
return render_template('chat_viewer.html', chat=chat)
|
||||
|
||||
@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_subchannel_messages():
|
||||
subchannel_id = request.args.get('id')
|
||||
instance_uuid = request.args.get('uuid')
|
||||
subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid)
|
||||
if subchannel[1] != 200:
|
||||
return create_json_response(subchannel[0], subchannel[1])
|
||||
else:
|
||||
subchannel = subchannel[0]
|
||||
return render_template('SubChannelMessages.html', subchannel=subchannel)
|
||||
|
||||
@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_message():
|
||||
message_id = request.args.get('id')
|
||||
message = chats_viewer.api_get_message(message_id)
|
||||
if message[1] != 200:
|
||||
return create_json_response(message[0], message[1])
|
||||
else:
|
||||
message = message[0]
|
||||
return render_template('ChatMessage.html', message=message)
|
||||
|
||||
#############################################################################################
|
||||
#############################################################################################
|
||||
#############################################################################################
|
||||
|
||||
|
||||
@chats_explorer.route("/objects/chat/messages", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_dashboard_chat():
|
||||
chat = request.args.get('id')
|
||||
subtype = request.args.get('subtype')
|
||||
chat = Chats.Chat(chat, subtype)
|
||||
if chat.exists():
|
||||
messages, mess_tags = chat.get_messages()
|
||||
print(messages)
|
||||
print(chat.get_subchannels())
|
||||
meta = chat.get_meta({'icon', 'username'})
|
||||
if meta.get('username'):
|
||||
meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta()
|
||||
print(meta)
|
||||
return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label)
|
||||
else:
|
||||
return abort(404)
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ail_core
|
||||
from lib.objects import abstract_subtype_object
|
||||
from lib.objects import ail_objects
|
||||
from lib.objects import Chats
|
||||
from packages import Date
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
def create_json_response(data, status_code):
|
||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
|
||||
@objects_chat.route("/objects/chat/messages", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_dashboard_chat():
|
||||
chat = request.args.get('id')
|
||||
subtype = request.args.get('subtype')
|
||||
chat = Chats.Chat(chat, subtype)
|
||||
if chat.exists():
|
||||
messages = chat.get_messages()
|
||||
meta = chat.get_meta({'icon'})
|
||||
print(meta)
|
||||
return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label)
|
||||
else:
|
||||
return abort(404)
|
||||
|
||||
|
||||
|
||||
209
var/www/templates/chats_explorer/ChatMessage.html
Normal file
209
var/www/templates/chats_explorer/ChatMessage.html
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Chat Messages - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.chat-message-left,
|
||||
.chat-message-right {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.chat-message-right {
|
||||
flex-direction: row-reverse;
|
||||
margin-left: auto
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h3 class="text-secondary">{{ meta["id"] }} :</h3>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Object subtype</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
<th>Username</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="{{ meta["icon"]["color"] }}"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta["icon"]["style"] }}" font-size="16px">{{ meta["icon"]["icon"] }}</text>
|
||||
</g>
|
||||
</svg>
|
||||
{{ meta["subtype"] }}
|
||||
</td>
|
||||
<td>{{ meta['first_seen'] }}</td>
|
||||
<td>{{ meta['last_seen'] }}</td>
|
||||
<td>
|
||||
{% if 'username' in meta %}
|
||||
{{ meta['username']['id'] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ meta['nb_seen'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-md-1">
|
||||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="list-group-item py-0">
|
||||
<br>
|
||||
<div class="mb-3">
|
||||
Tags:
|
||||
{% for tag in meta['tags'] %}
|
||||
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
|
||||
data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">
|
||||
{{ tag }}
|
||||
</button>
|
||||
{% endfor %}
|
||||
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
|
||||
<i class="far fa-plus-square"></i>
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
{% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}
|
||||
{% include 'modals/investigations_register_obj.html' %}
|
||||
{% endwith %}
|
||||
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||
<i class="fas fa-microscope"></i> Investigations
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% for tag in mess_tags %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
|
||||
{% endfor %}
|
||||
|
||||
<div>
|
||||
<div class="list-group d-inline-block">
|
||||
{% for date in messages %}
|
||||
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="position-relative">
|
||||
<div class="chat-messages p-4">
|
||||
|
||||
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
|
||||
|
||||
<div class="chat-message-left pb-1">
|
||||
<div>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
|
||||
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
|
||||
</div>
|
||||
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['user-account']['username'] %}
|
||||
{{ mess['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if mess['reply_to'] %}
|
||||
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['reply_to']['user-account']['username'] %}
|
||||
{{ mess['reply_to']['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['reply_to']['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
|
||||
{% for tag in mess['reply_to']['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
|
||||
{# <div class="">#}
|
||||
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
|
||||
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
|
||||
{# </div>#}
|
||||
</div>
|
||||
{% endif %}
|
||||
<pre class="my-0">{{ mess['content'] }}</pre>
|
||||
{% for tag in mess['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
<div class="">
|
||||
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
|
||||
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_chat").addClass("active");
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -58,6 +58,7 @@
|
|||
<th>Object subtype</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
<th>Username</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
|
@ -74,6 +75,11 @@
|
|||
</td>
|
||||
<td>{{ meta['first_seen'] }}</td>
|
||||
<td>{{ meta['last_seen'] }}</td>
|
||||
<td>
|
||||
{% if 'username' in meta %}
|
||||
{{ meta['username']['id'] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ meta['nb_seen'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
|
@ -111,11 +117,23 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% for tag in mess_tags %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
|
||||
{% endfor %}
|
||||
|
||||
<div>
|
||||
<div class="list-group d-inline-block">
|
||||
{% for date in messages %}
|
||||
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="position-relative">
|
||||
<div class="chat-messages p-4">
|
||||
|
||||
{% for date in messages %}
|
||||
<h2><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
|
||||
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
|
||||
{% for mess in messages[date] %}
|
||||
|
||||
<div class="chat-message-left pb-1">
|
||||
211
var/www/templates/chats_explorer/SubChannelMessages.html
Normal file
211
var/www/templates/chats_explorer/SubChannelMessages.html
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Sub-Channel Messages - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
{# <link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
{# <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.chat-message-left,
|
||||
.chat-message-right {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.chat-message-right {
|
||||
flex-direction: row-reverse;
|
||||
margin-left: auto
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h3 class="text-secondary">{{ subchannel["id"] }} :</h3>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Chat Instance</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
<th>Username</th>
|
||||
<th>Nb Messages</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
{{ subchannel["subtype"] }}
|
||||
</td>
|
||||
<td>{{ subchannel['first_seen'] }}</td>
|
||||
<td>{{ subchannel['last_seen'] }}</td>
|
||||
<td>
|
||||
{% if 'username' in subchannel %}
|
||||
{{ subchannel['username'] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ subchannel['nb_messages'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="list-group-item py-0">
|
||||
<br>
|
||||
<div class="mb-3">
|
||||
Tags:
|
||||
{% for tag in subchannel['tags'] %}
|
||||
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
|
||||
data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ subchannel["subtype"] }}" data-objid="{{ subchannel["id"] }}">
|
||||
{{ tag }}
|
||||
</button>
|
||||
{% endfor %}
|
||||
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
|
||||
<i class="far fa-plus-square"></i>
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
{% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}
|
||||
{% include 'modals/investigations_register_obj.html' %}
|
||||
{% endwith %}
|
||||
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||
<i class="fas fa-microscope"></i> Investigations
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% for tag in mess_tags %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
|
||||
{% endfor %}
|
||||
|
||||
<div>
|
||||
<div class="list-group d-inline-block">
|
||||
{% for date in messages %}
|
||||
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="position-relative">
|
||||
<div class="chat-messages p-4">
|
||||
|
||||
{% for date in subchannel['messages'] %}
|
||||
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
|
||||
{% for mess in subchannel['messages'][date] %}
|
||||
|
||||
<div class="chat-message-left pb-1">
|
||||
<div>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
|
||||
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
|
||||
</div>
|
||||
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['user-account']['username'] %}
|
||||
{{ mess['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if mess['reply_to'] %}
|
||||
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
|
||||
<div class="font-weight-bold mb-1">
|
||||
{% if mess['reply_to']['user-account']['username'] %}
|
||||
{{ mess['reply_to']['user-account']['username']['id'] }}
|
||||
{% else %}
|
||||
{{ mess['reply_to']['user-account']['id'] }}
|
||||
{% endif %}
|
||||
</div>
|
||||
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
|
||||
{% for tag in mess['reply_to']['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
|
||||
{# <div class="">#}
|
||||
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
|
||||
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
|
||||
{# </div>#}
|
||||
</div>
|
||||
{% endif %}
|
||||
<pre class="my-0">{{ mess['content'] }}</pre>
|
||||
{% for tag in mess['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
<div class="">
|
||||
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
|
||||
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endfor %}
|
||||
<br>
|
||||
{% endfor %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_chat").addClass("active");
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
123
var/www/templates/chats_explorer/chat_instance.html
Normal file
123
var/www/templates/chats_explorer/chat_instance.html
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Chats Protocols - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4 class="text-secondary">{{ chat_instance["protocol"] }} :</h4>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Chat Instance</th>
|
||||
<th>Network</th>
|
||||
<th>Address</th>
|
||||
<th>Nb Chats</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
{{ chat_instance["uuid"] }}
|
||||
</td>
|
||||
<td>{% if chat_instance["network"] %}{{ chat_instance["network"]}}{% endif %}</td>
|
||||
<td>{% if chat_instance["address"] %}{{ chat_instance["address"] }}{% endif %}</td>
|
||||
<td>{{ chat_instance["chats"] | length }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<table id="tablechats" class="table table-striped table-bordered">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Icon</th>
|
||||
<th>Name</th>
|
||||
<th>ID</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Chats</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for chat in chat_instance["chats"] %}
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>{{ chat['name'] }}</td>
|
||||
<td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>
|
||||
<td>{{ chat['first_seen'] }}</td>
|
||||
<td>{{ chat['last_seen'] }}</td>
|
||||
<td>{{ chat['nb_subchannels'] }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_cve").addClass("active");
|
||||
|
||||
$('#tablechats').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 4, "desc" ]]
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
133
var/www/templates/chats_explorer/chat_viewer.html
Normal file
133
var/www/templates/chats_explorer/chat_viewer.html
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Chats Protocols - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4 class="text-secondary">{{ chat["username"] }} {{ chat["id"] }} :</h4>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<table class="table">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Icon</th>
|
||||
<th>Name</th>
|
||||
<th>ID</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Sub-Channels</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>{{ chat['name'] }}</td>
|
||||
<td>{{ chat['id'] }}</td>
|
||||
<td>{{ chat['first_seen'] }}</td>
|
||||
<td>{{ chat['last_seen'] }}</td>
|
||||
<td>{{ chat['nb_subchannels'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if chat['subchannels'] %}
|
||||
<h4>Sub-Channels:</h4>
|
||||
<table id="tablesubchannels" class="table">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Icon</th>
|
||||
<th>Name</th>
|
||||
<th>ID</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Messages</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for meta in chat["subchannels"] %}
|
||||
<tr>
|
||||
<td>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
|
||||
</td>
|
||||
<td><b>{{ meta['name'] }}</b></td>
|
||||
<td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
|
||||
<td>{{ meta['first_seen'] }}</td>
|
||||
<td>{{ meta['last_seen'] }}</td>
|
||||
<td>{{ meta['nb_messages'] }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_cve").addClass("active");
|
||||
|
||||
{% if chat['subchannels'] %}
|
||||
$('#tablesubchannels').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 4, "desc" ]]
|
||||
});
|
||||
{% endif %}
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
80
var/www/templates/chats_explorer/chats_instance.html
Normal file
80
var/www/templates/chats_explorer/chats_instance.html
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Chats Protocols - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% if protocols %}
|
||||
<div class="d-flex justify-content-around my-2">
|
||||
{% for meta in protocols %}
|
||||
<a class="btn btn-light border-secondary" style="" href="#">
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">
|
||||
{% if meta['icon'] %}
|
||||
<i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
|
||||
{% endif %}
|
||||
<b>{{ meta['id'] }}</b>
|
||||
</h4>
|
||||
</div>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
<h4>No Protocol/Chats Imported</h4>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_cve").addClass("active");
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
80
var/www/templates/chats_explorer/chats_protocols.html
Normal file
80
var/www/templates/chats_explorer/chats_protocols.html
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Chats Protocols - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% if protocols %}
|
||||
<div class="d-flex justify-content-around my-2">
|
||||
{% for meta in protocols %}
|
||||
<a class="btn btn-light border-secondary" style="" href="#">
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">
|
||||
{% if meta['icon'] %}
|
||||
<i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
|
||||
{% endif %}
|
||||
<b>{{ meta['id'] }}</b>
|
||||
</h4>
|
||||
</div>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
<h4>No Protocol/Chats Imported</h4>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_cve").addClass("active");
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
Loading…
Reference in a new issue