mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 07:47:17 +00:00
Merge branch 'relationships' into otp
This commit is contained in:
terrtia
commit
8d4721d703
30 changed files with 1368 additions and 233 deletions
|
|
@ -44,6 +44,11 @@ class DefaultFeeder:
|
|||
def get_date(self):
|
||||
return datetime.date.today().strftime("%Y%m%d")
|
||||
|
||||
def get_feeder_timestamp(self):
|
||||
timestamp = self.json_data.get('timestamp')
|
||||
if timestamp:
|
||||
return int(timestamp)
|
||||
|
||||
def get_json_data(self):
|
||||
"""
|
||||
Return the JSON data,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
|
|||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from abc import ABC
|
||||
|
||||
|
|
@ -127,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
return self.json_data['meta'].get('reply_to', {}).get('message_id')
|
||||
|
||||
def get_message_forward(self):
|
||||
return self.json_data['meta'].get('forward')
|
||||
return self.json_data['meta'].get('forward', {})
|
||||
|
||||
def get_message_content(self):
|
||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||
|
|
@ -164,7 +165,40 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
self.obj = Messages.Message(obj_id)
|
||||
return self.obj
|
||||
|
||||
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
|
||||
# TODO handle subchannel
|
||||
def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
|
||||
chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
|
||||
|
||||
# Obj Daterange
|
||||
chat.add(date)
|
||||
|
||||
if meta_chat.get('name'):
|
||||
chat.set_name(meta_chat['name'])
|
||||
|
||||
if meta_chat.get('info'):
|
||||
chat.set_info(meta_chat['info'])
|
||||
|
||||
if meta_chat.get('date'): # TODO check if already exists
|
||||
chat.set_created_at(int(meta_chat['date']['timestamp']))
|
||||
|
||||
if meta_chat.get('icon'):
|
||||
img = Images.create(meta_chat['icon'], b64=True)
|
||||
img.add(date, chat)
|
||||
chat.set_icon(img.get_global_id())
|
||||
if new_objs:
|
||||
new_objs.add(img)
|
||||
|
||||
if meta_chat.get('username'):
|
||||
username = Username(meta_chat['username'], self.get_chat_protocol())
|
||||
chat.update_username_timeline(username.get_global_id(), int(time.time()))
|
||||
username.add(date, obj=chat) # TODO TODAY DATE
|
||||
|
||||
return chat
|
||||
|
||||
##############################################################################################################################
|
||||
|
||||
|
||||
def process_chat(self, new_objs, obj, date, timestamp, feeder_timestamp, reply_id=None):
|
||||
meta = self.json_data['meta']['chat'] # todo replace me by function
|
||||
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
||||
subchannel = None
|
||||
|
|
@ -190,7 +224,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
|
||||
if meta.get('username'):
|
||||
username = Username(meta['username'], self.get_chat_protocol())
|
||||
chat.update_username_timeline(username.get_global_id(), timestamp)
|
||||
chat.update_username_timeline(username.get_global_id(), feeder_timestamp)
|
||||
|
||||
if meta.get('subchannel'):
|
||||
subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
|
||||
|
|
@ -257,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
# else:
|
||||
# # ADD NEW MESSAGE REF (used by discord)
|
||||
|
||||
##########################################################################################
|
||||
|
||||
def _process_user(self, meta, date, timestamp, new_objs=None):
|
||||
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
|
||||
if meta.get('username'):
|
||||
username = Username(meta['username'], self.get_chat_protocol())
|
||||
# TODO timeline or/and correlation ????
|
||||
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||
user_account.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
||||
# Username---Message
|
||||
username.add(date) # TODO # correlation message ??? ###############################################################
|
||||
|
||||
# ADDITIONAL METAS
|
||||
if meta.get('firstname'):
|
||||
user_account.set_first_name(meta['firstname'])
|
||||
if meta.get('lastname'):
|
||||
user_account.set_last_name(meta['lastname'])
|
||||
if meta.get('phone'):
|
||||
user_account.set_phone(meta['phone'])
|
||||
|
||||
if meta.get('icon'):
|
||||
img = Images.create(meta['icon'], b64=True)
|
||||
img.add(date, user_account)
|
||||
user_account.set_icon(img.get_global_id())
|
||||
new_objs.add(img)
|
||||
|
||||
if meta.get('info'):
|
||||
user_account.set_info(meta['info'])
|
||||
|
||||
user_account.add(date)
|
||||
return user_account
|
||||
|
||||
def process_sender(self, new_objs, obj, date, timestamp):
|
||||
meta = self.json_data['meta'].get('sender')
|
||||
if not meta:
|
||||
|
|
@ -305,8 +372,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
if self.obj:
|
||||
objs.add(self.obj)
|
||||
new_objs = set()
|
||||
chats_objs = set()
|
||||
|
||||
date, timestamp = self.get_message_date_timestamp()
|
||||
feeder_timestamp = self.get_feeder_timestamp()
|
||||
if not feeder_timestamp:
|
||||
feeder_timestamp = timestamp
|
||||
|
||||
# REPLY
|
||||
reply_id = self.get_message_reply_id()
|
||||
|
|
@ -363,81 +434,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
print(obj.id)
|
||||
|
||||
# CHAT
|
||||
chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
|
||||
|
||||
# # TODO HANDLE OTHERS OBJECT TYPE
|
||||
# # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# # Message forward + Discussion
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# discussion_id = self.get_json_meta().get('discussion')
|
||||
# forward_from = self.get_message_forward()
|
||||
#
|
||||
# if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
|
||||
# chat_forward_id = forward_from['from']['id']
|
||||
# message_forward_id = forward_from['from']['channel_post']
|
||||
#
|
||||
# # if chat_forward_id == discussion_id:
|
||||
# # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
|
||||
# # if linked_chat.exists():
|
||||
# # # create thread
|
||||
# # # add message replies for each childrens
|
||||
#
|
||||
# # TODO HANDLE THREAD
|
||||
# # TODO Change FORWARD META FIELDS
|
||||
# # meta['forward'] = {}
|
||||
# # # CHAT ID
|
||||
# # # SUBCHANNEL ID -> can be None
|
||||
# # # Message ID
|
||||
#
|
||||
# # meta['forward']['origin']
|
||||
# # # same as 'forward'
|
||||
#
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# forward = self.get_message_forward()
|
||||
# f_chat = forward['chat']
|
||||
# f_subchannel = forward.get('subchannel')
|
||||
# f_id = forward.get('id')
|
||||
# if not f_subchannel:
|
||||
# chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
|
||||
# if chat_forward.exists():
|
||||
# for chat_obj in chat_objs:
|
||||
# if chat_obj.type == 'chat':
|
||||
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
|
||||
# # TODO LIST FORWARDED MESSAGES
|
||||
#
|
||||
#
|
||||
# # Discord -> serverID + subchannel ID + message ID
|
||||
# # Telegram -> chat ID + Message ID
|
||||
# # + ORIGIN IDs
|
||||
#
|
||||
#
|
||||
#
|
||||
# # TODO create relationships graph
|
||||
#
|
||||
#
|
||||
# # TODO REMOVE ME
|
||||
# # Message forward # TODO handle subchannel + message ID
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# forward_from = self.get_message_forward()
|
||||
# print('-----------------------------------------------------------')
|
||||
# print(forward_from)
|
||||
# if forward_from:
|
||||
# forward_from_type = forward_from['from']['type']
|
||||
# if forward_from_type == 'channel' or forward_from_type == 'chat':
|
||||
# chat_forward_id = forward_from['from']['id']
|
||||
# chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
|
||||
# if chat_forward.exists():
|
||||
# for chat_obj in chat_objs:
|
||||
# if chat_obj.type == 'chat':
|
||||
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
|
||||
# # chat_forward.add_relationship(obj.get_global_id(), 'forward')
|
||||
curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, feeder_timestamp, reply_id=reply_id)
|
||||
|
||||
# SENDER # TODO HANDLE NULL SENDER
|
||||
user_account = self.process_sender(new_objs, obj, date, timestamp)
|
||||
user_account = self.process_sender(new_objs, obj, date, feeder_timestamp)
|
||||
|
||||
if user_account:
|
||||
# UserAccount---ChatObjects
|
||||
for obj_chat in chat_objs:
|
||||
for obj_chat in curr_chats_objs:
|
||||
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
|
||||
|
||||
# if chat: # TODO Chat---Username correlation ???
|
||||
|
|
@ -449,4 +453,73 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
# -> subchannel ?
|
||||
# -> thread id ?
|
||||
|
||||
chats_objs.update(curr_chats_objs)
|
||||
|
||||
#######################################################################
|
||||
|
||||
## FORWARD ##
|
||||
chat_fwd = None
|
||||
user_fwd = None
|
||||
if self.get_json_meta().get('forward'):
|
||||
meta_fwd = self.get_message_forward()
|
||||
if meta_fwd.get('chat'):
|
||||
chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
|
||||
if meta_fwd.get('user'):
|
||||
user_fwd = self._process_user(meta_fwd['user'], date, feeder_timestamp, new_objs=new_objs) # TODO date, timestamp ???
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
|
||||
|
||||
# TODO chat_fwd -> message
|
||||
if chat_fwd or user_fwd:
|
||||
for obj in objs:
|
||||
if obj.type == 'message':
|
||||
if chat_fwd:
|
||||
obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
|
||||
if user_fwd:
|
||||
obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
obj.add_relationship(chat_obj.get_global_id(), 'in')
|
||||
# -FORWARD- #
|
||||
|
||||
## MENTION ##
|
||||
if self.get_json_meta().get('mentions'):
|
||||
for mention in self.get_json_meta()['mentions'].get('chats', []):
|
||||
m_obj = self._process_chat(mention, date, new_objs=new_objs)
|
||||
if m_obj:
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
|
||||
|
||||
# TODO PERF
|
||||
# TODO Keep message obj + chat obj in global var
|
||||
for obj in objs:
|
||||
if obj.type == 'message':
|
||||
obj.add_relationship(m_obj.get_global_id(), 'mention')
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
obj.add_relationship(chat_obj.get_global_id(), 'in')
|
||||
|
||||
for mention in self.get_json_meta()['mentions'].get('users', []):
|
||||
m_obj = self._process_user(mention, date, feeder_timestamp, new_objs=new_objs) # TODO date ???
|
||||
if m_obj:
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
|
||||
|
||||
# TODO PERF
|
||||
# TODO Keep message obj + chat obj in global var
|
||||
for obj in objs:
|
||||
if obj.type == 'message':
|
||||
obj.add_relationship(m_obj.get_global_id(), 'mention')
|
||||
for chat_obj in chats_objs:
|
||||
if chat_obj.type == 'chat':
|
||||
obj.add_relationship(chat_obj.get_global_id(), 'in')
|
||||
|
||||
# -MENTION- #
|
||||
|
||||
return new_objs | objs
|
||||
|
|
|
|||
|
|
@ -762,6 +762,9 @@ def delete_obj_trackers(obj_type, subtype, obj_id):
|
|||
#### TRACKERS ACL ####
|
||||
|
||||
## LEVEL ##
|
||||
def is_tracker_global_level(tracker_uuid):
|
||||
return int(r_tracker.hget(f'tracker:{tracker_uuid}', 'level')) == 1
|
||||
|
||||
def is_tracked_in_global_level(tracked, tracker_type):
|
||||
for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
|
||||
tracker = Tracker(tracker_uuid)
|
||||
|
|
@ -805,6 +808,19 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
|||
return {"status": "error", "reason": "Access Denied"}, 403
|
||||
return {"uuid": tracker_uuid}, 200
|
||||
|
||||
|
||||
def api_is_allowed_to_access_tracker(tracker_uuid, user_id):
|
||||
if not is_valid_uuid_v4(tracker_uuid):
|
||||
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||
tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
||||
if not tracker_creator:
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
user = User(user_id)
|
||||
if not is_tracker_global_level(tracker_uuid):
|
||||
if not user.is_in_role('admin') and user_id != tracker_creator:
|
||||
return {"status": "error", "reason": "Access Denied"}, 403
|
||||
return {"uuid": tracker_uuid}, 200
|
||||
|
||||
##-- ACL --##
|
||||
|
||||
#### FIX DB #### TODO ###################################################################
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
|
|||
if obj_type == 'chat-thread':
|
||||
return r_object.smembers(f'all_chat-thread:subtypes')
|
||||
if obj_type == 'cryptocurrency':
|
||||
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
|
||||
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
|
||||
if obj_type == 'pgp':
|
||||
return ['key', 'mail', 'name']
|
||||
if obj_type == 'username':
|
||||
|
|
|
|||
|
|
@ -328,6 +328,31 @@ def get_username_meta_from_global_id(username_global_id):
|
|||
username = Usernames.Username(username_id, instance_uuid)
|
||||
return username.get_meta(options={'icon'})
|
||||
|
||||
###############################################################################
|
||||
# TODO Pagination
|
||||
def list_messages_to_dict(l_messages_id, translation_target=None):
|
||||
options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
|
||||
meta = {}
|
||||
curr_date = None
|
||||
for mess_id in l_messages_id:
|
||||
message = Messages.Message(mess_id[1:])
|
||||
timestamp = message.get_timestamp()
|
||||
date_day = message.get_date()
|
||||
date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}'
|
||||
if date_day != curr_date:
|
||||
meta[date_day] = []
|
||||
curr_date = date_day
|
||||
meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
|
||||
meta[date_day].append(meta_mess)
|
||||
|
||||
# if mess_dict.get('tags'):
|
||||
# for tag in mess_dict['tags']:
|
||||
# if tag not in tags:
|
||||
# tags[tag] = 0
|
||||
# tags[tag] += 1
|
||||
# return messages, pagination, tags
|
||||
return meta
|
||||
|
||||
# TODO Filter
|
||||
## Instance type
|
||||
## Chats IDS
|
||||
|
|
@ -354,7 +379,7 @@ def get_messages_iterator(filters={}):
|
|||
# threads
|
||||
for threads in chat.get_threads():
|
||||
thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
|
||||
_, _ = thread._get_messages(nb=-1)
|
||||
messages, _ = thread._get_messages(nb=-1)
|
||||
for mess in messages:
|
||||
message_id, _, message_id = mess[0].split(':', )
|
||||
yield Messages.Message(message_id)
|
||||
|
|
@ -404,6 +429,15 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
|
|||
meta.append(chat_meta)
|
||||
return meta
|
||||
|
||||
def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
|
||||
meta = {}
|
||||
chat = Chats.Chat(chat_id, subtype)
|
||||
chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
|
||||
if chat_meta['username']:
|
||||
chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
|
||||
|
||||
meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None)
|
||||
return meta
|
||||
|
||||
def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
|
||||
week = {}
|
||||
|
|
@ -431,6 +465,60 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
|
|||
nb_day += 1
|
||||
return stats
|
||||
|
||||
def get_user_account_chats_chord(subtype, user_id):
|
||||
nb = {}
|
||||
user_account = UsersAccount.UserAccount(user_id, subtype)
|
||||
for chat_g_id in user_account.get_chats():
|
||||
c_subtype, c_id = chat_g_id.split(':', 1)
|
||||
chat = Chats.Chat(c_id, c_subtype)
|
||||
nb[f'chat:{chat_g_id}'] = len(chat.get_user_messages(user_id))
|
||||
|
||||
user_account_gid = user_account.get_global_id() # # #
|
||||
chord = {'meta': {}, 'data': []}
|
||||
label = get_chat_user_account_label(user_account_gid)
|
||||
if label:
|
||||
chord['meta'][user_account_gid] = label
|
||||
else:
|
||||
chord['meta'][user_account_gid] = user_account_gid
|
||||
|
||||
for chat_g_id in nb:
|
||||
label = get_chat_user_account_label(chat_g_id)
|
||||
if label:
|
||||
chord['meta'][chat_g_id] = label
|
||||
else:
|
||||
chord['meta'][chat_g_id] = chat_g_id
|
||||
chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
|
||||
return chord
|
||||
|
||||
def get_user_account_mentions_chord(subtype, user_id):
|
||||
chord = {'meta': {}, 'data': []}
|
||||
nb = {}
|
||||
user_account = UsersAccount.UserAccount(user_id, subtype)
|
||||
user_account_gid = user_account.get_global_id()
|
||||
label = get_chat_user_account_label(user_account_gid)
|
||||
if label:
|
||||
chord['meta'][user_account_gid] = label
|
||||
else:
|
||||
chord['meta'][user_account_gid] = user_account_gid
|
||||
|
||||
for mess in user_account.get_messages():
|
||||
m = Messages.Message(mess[9:])
|
||||
for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
|
||||
if rel:
|
||||
if not rel['target'] in nb:
|
||||
nb[rel['target']] = 0
|
||||
nb[rel['target']] += 1
|
||||
|
||||
for g_id in nb:
|
||||
label = get_chat_user_account_label(g_id)
|
||||
if label:
|
||||
chord['meta'][g_id] = label
|
||||
else:
|
||||
chord['meta'][g_id] = g_id
|
||||
chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
|
||||
return chord
|
||||
|
||||
|
||||
def _get_chat_card_meta_options():
|
||||
return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
|
||||
|
||||
|
|
@ -481,18 +569,65 @@ def fix_correlations_subchannel_message():
|
|||
|
||||
#### API ####
|
||||
|
||||
def get_chat_user_account_label(chat_gid):
|
||||
label = None
|
||||
obj_type, subtype, obj_id = chat_gid.split(':', 2)
|
||||
if obj_type == 'chat':
|
||||
obj = get_obj_chat(obj_type, subtype, obj_id)
|
||||
username = obj.get_username()
|
||||
if username:
|
||||
username = username.split(':', 2)[2]
|
||||
name = obj.get_name()
|
||||
if username and name:
|
||||
label = f'{username} - {name}'
|
||||
elif username:
|
||||
label = username
|
||||
elif name:
|
||||
label = name
|
||||
|
||||
elif obj_type == 'user-account':
|
||||
obj = UsersAccount.UserAccount(obj_id, subtype)
|
||||
username = obj.get_username()
|
||||
if username:
|
||||
username = username.split(':', 2)[2]
|
||||
name = obj.get_name()
|
||||
if username and name:
|
||||
label = f'{username} - {name}'
|
||||
elif username:
|
||||
label = username
|
||||
elif name:
|
||||
label = name
|
||||
return label
|
||||
|
||||
def enrich_chat_relationships_labels(relationships):
|
||||
meta = {}
|
||||
for row in relationships:
|
||||
if row['source'] not in meta:
|
||||
label = get_chat_user_account_label(row['source'])
|
||||
if label:
|
||||
meta[row['source']] = label
|
||||
else:
|
||||
meta[row['source']] = row['source']
|
||||
|
||||
if row['target'] not in meta:
|
||||
label = get_chat_user_account_label(row['target'])
|
||||
if label:
|
||||
meta[row['target']] = label
|
||||
else:
|
||||
meta[row['target']] = row['target']
|
||||
return meta
|
||||
def api_get_chat_service_instance(chat_instance_uuid):
|
||||
chat_instance = ChatServiceInstance(chat_instance_uuid)
|
||||
if not chat_instance.exists():
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
return chat_instance.get_meta({'chats'}), 200
|
||||
|
||||
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
|
||||
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True):
|
||||
chat = Chats.Chat(chat_id, chat_instance_uuid)
|
||||
if not chat.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
# print(chat.get_obj_language_stats())
|
||||
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
|
||||
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target)
|
||||
if meta['username']:
|
||||
meta['username'] = get_username_meta_from_global_id(meta['username'])
|
||||
if meta['subchannels']:
|
||||
|
|
@ -500,7 +635,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
|
|||
else:
|
||||
if translation_target not in Language.get_translation_languages():
|
||||
translation_target = None
|
||||
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
|
||||
if messages:
|
||||
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
|
||||
return meta, 200
|
||||
|
||||
def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id):
|
||||
|
|
@ -602,6 +738,18 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
|
|||
meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
|
||||
return meta, 200
|
||||
|
||||
def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None):
|
||||
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
|
||||
if not user_account.exists():
|
||||
return {"status": "error", "reason": "Unknown user-account"}, 404
|
||||
meta = get_user_account_chat_message(user_id, instance_uuid, chat_id)
|
||||
meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
|
||||
resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False)
|
||||
if resp[1] != 200:
|
||||
return resp
|
||||
meta['chat'] = resp[0]
|
||||
return meta, 200
|
||||
|
||||
def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
|
||||
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
|
||||
if not user_account.exists():
|
||||
|
|
|
|||
|
|
@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he
|
|||
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
|
||||
return schedule_uuid
|
||||
|
||||
def _delete_schedules():
|
||||
for schedule_uuid in get_schedulers_uuid():
|
||||
schedule = CrawlerSchedule(schedule_uuid)
|
||||
schedule.delete()
|
||||
|
||||
# TODO sanityze UUID
|
||||
def api_delete_schedule(data):
|
||||
schedule_uuid = data.get('uuid')
|
||||
|
|
@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
|
|||
external=external)
|
||||
return task_uuid
|
||||
|
||||
|
||||
## -- CRAWLER TASK -- ##
|
||||
|
||||
#### CRAWLER TASK API ####
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject):
|
|||
pass
|
||||
|
||||
def is_valid_address(self):
|
||||
if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin':
|
||||
if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron':
|
||||
return check_base58_address(self.id)
|
||||
else:
|
||||
return True
|
||||
|
|
@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject):
|
|||
return 'ZEC'
|
||||
elif self.subtype == 'dash':
|
||||
return 'DASH'
|
||||
elif self.subtype == 'tron':
|
||||
return 'TRX'
|
||||
return None
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
|
|
@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject):
|
|||
|
||||
def get_all_subtypes():
|
||||
# return ail_core.get_object_all_subtypes(self.type)
|
||||
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
|
||||
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
|
||||
|
||||
|
||||
# def build_crypto_regex(subtype, search_id):
|
||||
|
|
@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol):
|
|||
return 'zcash'
|
||||
elif symbol == 'DASH':
|
||||
return 'dash'
|
||||
elif symbol == 'TRX':
|
||||
return 'tron'
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype):
|
|||
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
|
||||
if subtype == '':
|
||||
pass
|
||||
elif subtype == 'name':
|
||||
pass
|
||||
elif subtype == 'mail':
|
||||
pass
|
||||
return name_to_search
|
||||
|
||||
def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
|
||||
|
|
|
|||
|
|
@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
|
|||
def get_last_name(self):
|
||||
return self._get_field('lastname')
|
||||
|
||||
def get_name(self):
|
||||
first_name = self.get_first_name()
|
||||
last_name = self.get_last_name()
|
||||
if first_name and last_name:
|
||||
return f'{first_name} {last_name}'
|
||||
elif first_name:
|
||||
return first_name
|
||||
elif last_name:
|
||||
return last_name
|
||||
|
||||
def get_phone(self):
|
||||
return self._get_field('phone')
|
||||
|
||||
|
|
@ -129,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
|
|||
|
||||
def get_messages(self):
|
||||
messages = []
|
||||
for mess in self.get_correlation('message'):
|
||||
messages.append(f'message:{mess}')
|
||||
correl = self.get_correlation('message')
|
||||
if 'message' in correl:
|
||||
for mess_id in correl['message']:
|
||||
messages.append(f'message:{mess_id}')
|
||||
return messages
|
||||
|
||||
|
||||
def get_messages_by_chat_obj(self, chat_obj):
|
||||
messages = []
|
||||
for mess in self.get_correlation_iter_obj(chat_obj, 'message'):
|
||||
|
|
@ -157,6 +168,7 @@ class UserAccount(AbstractSubtypeObject):
|
|||
meta['usernames'] = self.get_usernames()
|
||||
if 'icon' in options:
|
||||
meta['icon'] = self.get_icon()
|
||||
meta['svg_icon'] = meta['icon']
|
||||
if 'info' in options:
|
||||
meta['info'] = self.get_info()
|
||||
if 'translation' in options and translation_target:
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
|
|||
from lib import Duplicate
|
||||
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
|
||||
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
|
||||
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
|
||||
from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
|
||||
from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
|
||||
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
|
||||
|
||||
|
|
@ -299,6 +299,9 @@ class AbstractObject(ABC):
|
|||
def get_nb_relationships(self, filter=[]):
|
||||
return get_obj_nb_relationships(self.get_global_id())
|
||||
|
||||
def get_obj_relationships(self, relationships=set(), filter_types=set()):
|
||||
return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
|
||||
|
||||
def add_relationship(self, obj2_global_id, relationship, source=True):
|
||||
# is source
|
||||
if source:
|
||||
|
|
|
|||
|
|
@ -562,24 +562,47 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
|
|||
|
||||
# --- CORRELATION --- #
|
||||
|
||||
#### RELATIONSHIPS ####
|
||||
|
||||
def get_relationships():
|
||||
return relationships_engine.get_relationships()
|
||||
|
||||
def sanitize_relationships(relationships):
|
||||
return relationships_engine.sanitize_relationships(relationships)
|
||||
def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_nb_relationships(filter=filter_types)
|
||||
|
||||
def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
|
||||
def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
|
||||
objs_hidden=set(),
|
||||
flask_context=False):
|
||||
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
|
||||
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
|
||||
filter_types=filter_types,
|
||||
max_nodes=max_nodes, level=level,
|
||||
objs_hidden=objs_hidden)
|
||||
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
|
||||
filter_types=filter_types,
|
||||
max_nodes=max_nodes, level=level,
|
||||
objs_hidden=objs_hidden)
|
||||
# print(meta)
|
||||
meta['objs'] = list(meta['objs'])
|
||||
return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
|
||||
"links": links,
|
||||
"meta": meta}
|
||||
|
||||
def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
|
||||
if obj_type == 'chat':
|
||||
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
|
||||
data = relationships_engine.get_chat_forward_stats(obj_global_id)
|
||||
return data
|
||||
return []
|
||||
|
||||
def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
|
||||
if obj_type == 'chat':
|
||||
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
|
||||
data = relationships_engine.get_chat_mentions_stats(obj_global_id)
|
||||
return data
|
||||
return []
|
||||
|
||||
# --- RELATIONSHIPS --- #
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
|
@ -16,41 +16,103 @@ config_loader = None
|
|||
|
||||
|
||||
RELATIONSHIPS = {
|
||||
"forward",
|
||||
"forwarded_from",
|
||||
"forwarded_to", # forwarded_to
|
||||
"in",
|
||||
"mention"
|
||||
}
|
||||
|
||||
RELATIONSHIPS_OBJS = { # TODO forward user-account
|
||||
"forwarded_from": {
|
||||
'chat': {'message'},
|
||||
'message': {'chat', 'user-account'}
|
||||
},
|
||||
"forwarded_to": {
|
||||
'chat': {'chat'},
|
||||
'user-account': {'chat'},
|
||||
},
|
||||
"in": {
|
||||
'chat': {'message'},
|
||||
'message': {'chat'}
|
||||
},
|
||||
"mention": {
|
||||
'chat': {'chat', 'user-account', 'message'},
|
||||
'message': {'chat', 'user-account'},
|
||||
'user-account': {'chat', 'message'},
|
||||
},
|
||||
}
|
||||
|
||||
def get_relationships():
|
||||
return RELATIONSHIPS
|
||||
|
||||
|
||||
def get_obj_relationships_by_type(obj_global_id, relationship):
|
||||
return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
|
||||
|
||||
def get_obj_nb_relationships_by_type(obj_global_id, relationship):
|
||||
return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
|
||||
|
||||
def get_obj_relationships(obj_global_id):
|
||||
relationships = []
|
||||
for relationship in get_relationships():
|
||||
for rel in get_obj_relationships_by_type(obj_global_id, relationship):
|
||||
meta = {'relationship': relationship}
|
||||
direction, obj_id = rel.split(':', 1)
|
||||
if direction == 'i':
|
||||
meta['source'] = obj_id
|
||||
meta['target'] = obj_global_id
|
||||
else:
|
||||
meta['target'] = obj_id
|
||||
meta['source'] = obj_global_id
|
||||
|
||||
if not obj_id.startswith('chat'):
|
||||
continue
|
||||
|
||||
meta['id'] = obj_id
|
||||
# meta['direction'] = direction
|
||||
relationships.append(meta)
|
||||
def sanitize_relationships(relationships):
|
||||
rels = get_relationships()
|
||||
if relationships:
|
||||
relationships = set(relationships).intersection(rels)
|
||||
if not relationships:
|
||||
relationships = rels
|
||||
if not relationships:
|
||||
return []
|
||||
return relationships
|
||||
|
||||
def get_obj_nb_relationships(obj_global_id):
|
||||
def get_relationship_obj_types(relationship):
|
||||
return RELATIONSHIPS_OBJS.get(relationship, {})
|
||||
|
||||
def get_relationship_objs(relationship, obj_type):
|
||||
return get_relationship_obj_types(relationship).get(obj_type, set())
|
||||
|
||||
def sanityze_obj_types(relationship, obj_type, filter_types):
|
||||
objs_types = get_relationship_objs(relationship, obj_type)
|
||||
if filter_types:
|
||||
filter_types = objs_types.intersection(filter_types)
|
||||
else:
|
||||
filter_types = objs_types
|
||||
# if not filter_types:
|
||||
# filter_types = objs_types
|
||||
# if not filter_types:
|
||||
# return []
|
||||
return filter_types
|
||||
|
||||
# TODO check obj_type
|
||||
# TODO sanitize relationships
|
||||
|
||||
def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
|
||||
obj_type = obj_global_id.split(':', 1)[0]
|
||||
relationships = {}
|
||||
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
|
||||
for o_type in filter_types:
|
||||
relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
|
||||
return relationships
|
||||
|
||||
def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
|
||||
obj_type = obj_global_id.split(':', 1)[0]
|
||||
relationships = {}
|
||||
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
|
||||
for o_type in filter_types:
|
||||
relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
|
||||
return relationships
|
||||
|
||||
def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
|
||||
all_relationships = []
|
||||
for relationship in relationships:
|
||||
obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
|
||||
for obj_type in obj_relationships:
|
||||
for rel in obj_relationships[obj_type]:
|
||||
meta = {'relationship': relationship}
|
||||
direction, obj_id = rel.split(':', 1)
|
||||
if direction == 'i':
|
||||
meta['source'] = obj_id
|
||||
meta['target'] = obj_global_id
|
||||
else:
|
||||
meta['target'] = obj_id
|
||||
meta['source'] = obj_global_id
|
||||
|
||||
meta['id'] = obj_id
|
||||
# meta['direction'] = direction
|
||||
all_relationships.append(meta)
|
||||
return all_relationships
|
||||
|
||||
def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
|
||||
nb = {}
|
||||
for relationship in get_relationships():
|
||||
nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
|
||||
|
|
@ -59,27 +121,27 @@ def get_obj_nb_relationships(obj_global_id):
|
|||
|
||||
# TODO Filter by obj type ???
|
||||
def add_obj_relationship(source, target, relationship):
|
||||
r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
|
||||
r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
|
||||
# r_rel.sadd(f'rels:{source}', relationship)
|
||||
# r_rel.sadd(f'rels:{target}', relationship)
|
||||
source_type = source.split(':', 1)[0]
|
||||
target_type = target.split(':', 1)[0]
|
||||
r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
|
||||
r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
|
||||
|
||||
|
||||
def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
|
||||
def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
|
||||
links = []
|
||||
nodes = set()
|
||||
meta = {'complete': True, 'objs': set()}
|
||||
done = set()
|
||||
done_link = set()
|
||||
|
||||
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
return nodes, links, meta
|
||||
|
||||
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
|
||||
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
|
||||
meta['objs'].add(obj_global_id)
|
||||
nodes.add(obj_global_id)
|
||||
|
||||
for rel in get_obj_relationships(obj_global_id):
|
||||
for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
|
||||
meta['objs'].add(rel['id'])
|
||||
|
||||
if rel['id'] in done:
|
||||
|
|
@ -99,13 +161,119 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
|
|||
if level > 0:
|
||||
next_level = level - 1
|
||||
|
||||
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
|
||||
# done.add(rel['id'])
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
|
||||
|
||||
if __name__ == '__main__':
|
||||
source = ''
|
||||
target = ''
|
||||
add_obj_relationship(source, target, 'forward')
|
||||
# print(get_obj_relationships(source))
|
||||
def get_chat_forward_stats_out(obj_global_id):
|
||||
nb = {}
|
||||
for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
|
||||
chat_mess = rel['source'].split('/')
|
||||
|
||||
chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||
if chat_target not in nb:
|
||||
nb[chat_target] = 0
|
||||
nb[chat_target] += 1
|
||||
return nb
|
||||
|
||||
def get_chat_forward_stats_in(obj_global_id):
|
||||
nb = {}
|
||||
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
|
||||
r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
|
||||
if r:
|
||||
if not r[0]['target'] in nb:
|
||||
nb[r[0]['target']] = 0
|
||||
nb[r[0]['target']] += 1
|
||||
# chat_mess = rel['source'].split('/')
|
||||
#
|
||||
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||
# print(chat_target, chat, chat_target == chat)
|
||||
# if chat is None or chat_target == chat:
|
||||
# if chat_target not in nb:
|
||||
# nb[chat_target] = 0
|
||||
# nb[chat_target] += 1
|
||||
#
|
||||
# print(json.dumps(nb, indent=4))
|
||||
return nb
|
||||
|
||||
def get_chat_forward_stats(obj_global_id): # objs_hidden
|
||||
data = []
|
||||
|
||||
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
|
||||
|
||||
in_mess = get_chat_forward_stats_out(obj_global_id)
|
||||
|
||||
out_mess = get_chat_forward_stats_in(obj_global_id)
|
||||
|
||||
|
||||
# if rel['target'] == obj_global_id:
|
||||
# # print(rel)
|
||||
# nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
|
||||
#
|
||||
for target in in_mess:
|
||||
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
|
||||
|
||||
for source in out_mess:
|
||||
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
|
||||
|
||||
# print()
|
||||
# print(in_mess)
|
||||
# print()
|
||||
# print(out_mess)
|
||||
|
||||
return data
|
||||
|
||||
##################################################################
|
||||
|
||||
def get_chat_mention_stats_in(obj_global_id):
|
||||
nb = {}
|
||||
for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
|
||||
chat_mess = rel['source'].split('/')
|
||||
chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||
if chat_source not in nb:
|
||||
nb[chat_source] = 0
|
||||
nb[chat_source] += 1
|
||||
return nb
|
||||
|
||||
def get_chat_mention_stats_out(obj_global_id):
|
||||
nb = {}
|
||||
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
|
||||
r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
|
||||
if r:
|
||||
if not r[0]['target'] in nb:
|
||||
nb[r[0]['target']] = 0
|
||||
nb[r[0]['target']] += 1
|
||||
# chat_mess = rel['source'].split('/')
|
||||
#
|
||||
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||
# print(chat_target, chat, chat_target == chat)
|
||||
# if chat is None or chat_target == chat:
|
||||
# if chat_target not in nb:
|
||||
# nb[chat_target] = 0
|
||||
# nb[chat_target] += 1
|
||||
#
|
||||
# print(json.dumps(nb, indent=4))
|
||||
return nb
|
||||
|
||||
def get_chat_mentions_stats(obj_global_id): # objs_hidden
|
||||
data = []
|
||||
|
||||
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
|
||||
|
||||
out_mess = get_chat_mention_stats_in(obj_global_id)
|
||||
in_mess = get_chat_mention_stats_out(obj_global_id)
|
||||
#
|
||||
for target in in_mess:
|
||||
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
|
||||
|
||||
for source in out_mess:
|
||||
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
|
||||
|
||||
# print()
|
||||
# print(in_mess)
|
||||
# print()
|
||||
# print(out_mess)
|
||||
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -92,7 +92,13 @@ CURRENCIES = {
|
|||
'regex': r'\b(?<![+/=])X[A-Za-z0-9]{33}(?![+/=])\b',
|
||||
'max_execution_time': default_max_execution_time,
|
||||
'tag': 'infoleak:automatic-detection="dash-address"',
|
||||
}
|
||||
},
|
||||
'tron': {
|
||||
'name': 'tron', # e.g. TYdds9VLDjUshf9tbsXSfGUZNzJSbbBeat
|
||||
'regex': r'\b(?<![+/=])T[0-9a-zA-Z]{33}(?![+/=])\b',
|
||||
'max_execution_time': default_max_execution_time,
|
||||
'tag': 'infoleak:automatic-detection="tron-address"',
|
||||
},
|
||||
}
|
||||
##################################
|
||||
##################################
|
||||
|
|
|
|||
|
|
@ -263,3 +263,9 @@ def sanitise_daterange(date_from, date_to, separator='', date_type='str'):
|
|||
date_from = date_to
|
||||
date_to = res
|
||||
return date_from, date_to
|
||||
|
||||
def get_previous_month_date():
|
||||
now = datetime.date.today()
|
||||
first = now.replace(day=1)
|
||||
last_month = first - datetime.timedelta(days=1)
|
||||
return last_month.strftime("%Y%m%d")
|
||||
|
|
|
|||
|
|
@ -297,6 +297,44 @@ def objects_user_account():
|
|||
ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
|
||||
translation_languages=languages, translation_target=target)
|
||||
|
||||
@chats_explorer.route("/objects/user-account_chats_chord_json", methods=['GET']) # TODO API
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_user_account_chats_chord_json():
|
||||
subtype = request.args.get('subtype')
|
||||
user_id = request.args.get('id')
|
||||
json_graph = chats_viewer.get_user_account_chats_chord(subtype, user_id)
|
||||
return jsonify(json_graph)
|
||||
|
||||
@chats_explorer.route("/objects/user-account_mentions_chord_json", methods=['GET']) # TODO API
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_user_account_mentions_chord_json():
|
||||
subtype = request.args.get('subtype')
|
||||
user_id = request.args.get('id')
|
||||
json_graph = chats_viewer.get_user_account_mentions_chord(subtype, user_id)
|
||||
return jsonify(json_graph)
|
||||
|
||||
@chats_explorer.route("/objects/user-account/chat", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_user_account_chat():
|
||||
instance_uuid = request.args.get('subtype')
|
||||
user_id = request.args.get('id')
|
||||
chat_id = request.args.get('chat_id')
|
||||
target = request.args.get('target')
|
||||
if target == "Don't Translate":
|
||||
target = None
|
||||
meta = chats_viewer.api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=target)
|
||||
if meta[1] != 200:
|
||||
return create_json_response(meta[0], meta[1])
|
||||
else:
|
||||
meta = meta[0]
|
||||
languages = Language.get_translation_languages()
|
||||
return render_template('chats_explorer/user_chat_messages.html', meta=meta, bootstrap_label=bootstrap_label,
|
||||
ail_tags=Tag.get_modal_add_tags(meta['user-account']['id'], meta['user-account']['type'], meta['user-account']['subtype']),
|
||||
translation_languages=languages, translation_target=target)
|
||||
|
||||
@chats_explorer.route("objects/user-account/messages/stats/week/all", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects import ail_objects
|
||||
from lib import chats_viewer
|
||||
from lib import Tag
|
||||
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
|
@ -263,9 +264,37 @@ def relationships_graph_node_json():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
|
||||
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
|
||||
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
|
||||
|
||||
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
|
||||
return jsonify(json_graph)
|
||||
|
||||
@correlation.route('/relationships/chord_graph_json')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def relationships_chord_graph_json():
|
||||
obj_id = request.args.get('id')
|
||||
subtype = request.args.get('subtype')
|
||||
obj_type = request.args.get('type')
|
||||
|
||||
chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
|
||||
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
|
||||
|
||||
return jsonify({'meta': meta, 'data': chat_json_graph})
|
||||
|
||||
@correlation.route('/relationships/chord_mentions_graph_json')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def relationships_chord_mentions_graph_json():
|
||||
obj_id = request.args.get('id')
|
||||
subtype = request.args.get('subtype')
|
||||
obj_type = request.args.get('type')
|
||||
|
||||
chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
|
||||
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
|
||||
|
||||
return jsonify({'meta': meta, 'data': chat_json_graph})
|
||||
|
||||
@correlation.route('/relationship/show', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
|
|
@ -278,8 +307,28 @@ def show_relationship():
|
|||
max_nodes = request.form.get('max_nb_nodes_in')
|
||||
level = sanitise_level(request.form.get('level'))
|
||||
|
||||
## get all selected relationships
|
||||
relationships = []
|
||||
for relationship in ail_objects.get_relationships():
|
||||
rel_option = request.form.get(f'relationship_{relationship}_Check')
|
||||
if rel_option:
|
||||
relationships.append(relationship)
|
||||
|
||||
relationships = ",".join(relationships)
|
||||
|
||||
## get all selected objects types
|
||||
filter_types = []
|
||||
for ob_type in ail_objects.get_all_objects():
|
||||
correl_option = request.form.get(f'{ob_type}_Check')
|
||||
if correl_option:
|
||||
filter_types.append(ob_type)
|
||||
|
||||
# list as params
|
||||
filter_types = ",".join(filter_types)
|
||||
|
||||
# redirect to keep history and bookmark
|
||||
return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
|
||||
filter=filter_types, relationships=relationships,
|
||||
max_nodes=max_nodes, level=level))
|
||||
|
||||
# request.method == 'GET'
|
||||
|
|
@ -290,6 +339,9 @@ def show_relationship():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
|
||||
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
|
||||
|
||||
# check if obj_id exist
|
||||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
return abort(404)
|
||||
|
|
@ -300,6 +352,8 @@ def show_relationship():
|
|||
"object_type": obj_type,
|
||||
"max_nodes": max_nodes, "level": level,
|
||||
"correlation_id": obj_id,
|
||||
"relationships": relationships, "relationships_str": ",".join(relationships),
|
||||
"filter": filter_types, "filter_str": ",".join(filter_types),
|
||||
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
|
||||
"nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -316,6 +316,17 @@ def crawlers_last_domains_month_json():
|
|||
stats = crawlers.get_crawlers_stats_by_month(domain_type)
|
||||
return jsonify(stats)
|
||||
|
||||
@crawler_splash.route('/crawlers/last/domains/month/previous/json')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def crawlers_last_domains_previous_month_json():
|
||||
domain_type = request.args.get('type')
|
||||
if domain_type not in crawlers.get_crawler_all_types():
|
||||
return jsonify({'error': 'Invalid domain type'}), 400
|
||||
date = Date.get_previous_month_date()
|
||||
stats = crawlers.get_crawlers_stats_by_month(domain_type, date=date)
|
||||
return jsonify(stats)
|
||||
|
||||
@crawler_splash.route('/crawlers/last/domains/status/month/json')
|
||||
@login_required
|
||||
@login_read_only
|
||||
|
|
@ -800,7 +811,7 @@ def crawler_cookiejar_cookie_edit_post():
|
|||
@login_required
|
||||
@login_read_only
|
||||
def crawler_cookiejar_cookie_add():
|
||||
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
||||
cookiejar_uuid = request.args.get('uuid')
|
||||
return render_template("add_cookie.html", cookiejar_uuid=cookiejar_uuid)
|
||||
|
||||
|
||||
|
|
@ -831,7 +842,7 @@ def crawler_cookiejar_cookie_manual_add_post():
|
|||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
|
||||
return redirect(url_for('crawler_splash.crawler_cookiejar_show', uuid=cookiejar_uuid))
|
||||
|
||||
|
||||
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['POST'])
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ def tracked_menu_admin():
|
|||
def show_tracker():
|
||||
user_id = current_user.get_user_id()
|
||||
tracker_uuid = request.args.get('uuid', None)
|
||||
res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
|
||||
res = Tracker.api_is_allowed_to_access_tracker(tracker_uuid, user_id)
|
||||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ def show_tracker():
|
|||
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
|
||||
'user', 'webhook', 'nb_objs'})
|
||||
'user', 'webhooks', 'nb_objs'})
|
||||
|
||||
if meta['type'] == 'yara':
|
||||
yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
|
||||
|
|
@ -300,6 +300,7 @@ def add_tracked_menu():
|
|||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
return render_template("tracker_add.html",
|
||||
dict_tracker={},
|
||||
all_sources=item_basic.get_all_items_sources(r_list=True),
|
||||
tags_selector_data=Tag.get_tags_selector_data(),
|
||||
all_yara_files=Tracker.get_all_default_yara_files())
|
||||
|
|
@ -314,6 +315,8 @@ def tracker_edit():
|
|||
res = Tracker.api_edit_tracker(input_dict, user_id)
|
||||
if res[1] == 200:
|
||||
return redirect(url_for('hunters.show_tracker', uuid=res[0].get('uuid')))
|
||||
else:
|
||||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
user_id = current_user.get_user_id()
|
||||
tracker_uuid = request.args.get('uuid', None)
|
||||
|
|
@ -322,10 +325,16 @@ def tracker_edit():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhook'})
|
||||
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhooks'})
|
||||
if dict_tracker['type'] == 'yara':
|
||||
if not Tracker.is_default_yara_rule(dict_tracker['tracked']):
|
||||
dict_tracker['content'] = Tracker.get_yara_rule_content(dict_tracker['tracked'])
|
||||
elif dict_tracker['type'] == 'set':
|
||||
tracked, nb_words = dict_tracker['tracked'].rsplit(';', 1)
|
||||
tracked = tracked.replace(',', ' ')
|
||||
dict_tracker['tracked'] = tracked
|
||||
dict_tracker['nb_words'] = nb_words
|
||||
|
||||
taxonomies_tags, galaxies_tags, custom_tags = Tag.sort_tags_taxonomies_galaxies_customs(dict_tracker['tags'])
|
||||
tags_selector_data = Tag.get_tags_selector_data()
|
||||
tags_selector_data['taxonomies_tags'] = taxonomies_tags
|
||||
|
|
|
|||
140
var/www/static/js/d3/chord_directed_diagram.js
vendored
Normal file
140
var/www/static/js/d3/chord_directed_diagram.js
vendored
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
//Requirement: - D3v7
|
||||
// - jquery
|
||||
|
||||
// container_id = #container_id{"meta": {obj_gid: label, ...}, "data": [{"source": source, "target": target, "value": value}, ...]}
|
||||
// tooltip = d3 tooltip object
|
||||
|
||||
// TODO: - Mouseover object
|
||||
|
||||
const create_directed_chord_diagram = (container_id, data, min_value= 0, max_value = -1, fct_mouseover, fct_mouseout, options) => {
|
||||
|
||||
// Filter data by value between target and source
|
||||
if (min_value > 0) {
|
||||
data.data = data.data.filter(function(value) {
|
||||
return data.value >= min_value;
|
||||
});
|
||||
}
|
||||
if (max_value > 0) {
|
||||
data.data = data.data.filter(function(value) {
|
||||
return data.value <= max_value;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
function getMaxCharsToShow(angle, radius) {
|
||||
const approximateCharWidth = 7; // Approximate width of a character in pixels
|
||||
const arcLength = angle * radius; // Length of the arc
|
||||
return Math.floor(arcLength / approximateCharWidth); // Maximum number of characters that can fit
|
||||
}
|
||||
|
||||
const width = 900;
|
||||
const height = width;
|
||||
const innerRadius = Math.min(width, height) * 0.5 - 20;
|
||||
const outerRadius = innerRadius + 6;
|
||||
|
||||
const labels_meta = data.meta
|
||||
data = data.data
|
||||
|
||||
// Compute a dense matrix from the weighted links in data.
|
||||
var names = Array.from(d3.union(data.flatMap(d => [d.source, d.target])));
|
||||
const index = new Map(names.map((name, i) => [name, i]));
|
||||
const matrix = Array.from(index, () => new Array(names.length).fill(0));
|
||||
for (const {source, target, value} of data) matrix[index.get(source)][index.get(target)] += value;
|
||||
|
||||
const chord = d3.chordDirected()
|
||||
.padAngle(12 / innerRadius)
|
||||
.sortSubgroups(d3.descending)
|
||||
.sortChords(d3.descending);
|
||||
|
||||
const arc = d3.arc()
|
||||
.innerRadius(innerRadius)
|
||||
.outerRadius(outerRadius);
|
||||
|
||||
const ribbon = d3.ribbonArrow()
|
||||
.radius(innerRadius - 0.5)
|
||||
.padAngle(1 / innerRadius);
|
||||
|
||||
const colors = d3.schemeCategory10;
|
||||
|
||||
const svg = d3.select(container_id)
|
||||
.append("svg")
|
||||
.attr("width", width)
|
||||
.attr("height", height)
|
||||
.attr("viewBox", [-width / 2, -height / 2, width, height])
|
||||
.attr("style", "width: 100%; height: auto; font: 10px sans-serif;");
|
||||
|
||||
const chords = chord(matrix);
|
||||
|
||||
const textId = `text-${Math.random().toString(36).substring(2, 15)}`;
|
||||
|
||||
svg.append("path")
|
||||
.attr("id", textId)
|
||||
.attr("fill", "none")
|
||||
.attr("d", d3.arc()({outerRadius, startAngle: 0, endAngle: 2 * Math.PI}));
|
||||
|
||||
svg.append("g")
|
||||
.attr("fill-opacity", 0.75)
|
||||
.selectAll("path")
|
||||
.data(chords)
|
||||
.enter()
|
||||
.append("path")
|
||||
.attr("d", ribbon)
|
||||
.attr("fill", d => colors[d.target.index])
|
||||
.style("mix-blend-mode", "multiply")
|
||||
.append("title")
|
||||
.text(d => `${labels_meta[names[d.source.index]]}
|
||||
=> ${d.source.value} Messages
|
||||
${labels_meta[names[d.target.index]]}`);
|
||||
|
||||
const g = svg.append("g")
|
||||
.selectAll("g")
|
||||
.data(chords.groups)
|
||||
.enter()
|
||||
.append("g");
|
||||
|
||||
g.append("path")
|
||||
.attr("d", arc)
|
||||
.attr("fill", d => colors[d.index])
|
||||
.attr("stroke", "#fff")
|
||||
.on("mouseover", function(event, d) {
|
||||
fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]]);
|
||||
})
|
||||
.on("mouseout", fct_mouseout);
|
||||
|
||||
g.each(function(d) {
|
||||
const group = d3.select(this);
|
||||
const angle = d.endAngle - d.startAngle;
|
||||
const text = labels_meta[names[d.index]];
|
||||
const maxCharsToShow = getMaxCharsToShow(angle, outerRadius);
|
||||
|
||||
let displayedText
|
||||
if (maxCharsToShow <= 1) {
|
||||
displayedText = text[0];
|
||||
} else {
|
||||
displayedText = text.length > maxCharsToShow ? text.slice(0, maxCharsToShow - 1) + "…" : text;
|
||||
}
|
||||
|
||||
let info_text = "OUT: " + d3.sum(matrix[d.index]) + " ; IN: " + d3.sum(matrix, row => row[d.index]) + " Messages"
|
||||
|
||||
if (displayedText) {
|
||||
group.append("text")
|
||||
.attr("dy", -3)
|
||||
.append("textPath")
|
||||
.attr("xlink:href", `#${textId}`)
|
||||
.attr("startOffset", d.startAngle * outerRadius)
|
||||
.on("mouseover", function(event, d) {
|
||||
fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]], info_text);
|
||||
})
|
||||
.on("mouseout", fct_mouseout)
|
||||
.text(displayedText);
|
||||
}
|
||||
});
|
||||
|
||||
//return svg.node();
|
||||
}
|
||||
|
||||
|
||||
|
||||
// return svg ???
|
||||
// return svg
|
||||
|
||||
69
var/www/static/js/d3/heatmap_week_hour.js
vendored
69
var/www/static/js/d3/heatmap_week_hour.js
vendored
|
|
@ -1,4 +1,4 @@
|
|||
//Requirement: - D3v5
|
||||
//Requirement: - D3v7
|
||||
// - jquery
|
||||
// data input: var data = [{"count":0,"date":"2023-11-20","day":0,"hour":0}
|
||||
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
|
||||
|
|
@ -33,10 +33,12 @@ const create_heatmap_week_hour = (container_id, data, options) => {
|
|||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
// create a tooltip
|
||||
const tooltip = d3.select(container_id)
|
||||
const tooltip = d3.select("body")
|
||||
.append("div")
|
||||
.attr("class", "tooltip_heatmap")
|
||||
.style("opacity", 0)
|
||||
.attr("class", "tooltip")
|
||||
.style("position", "absolute")
|
||||
.style("pointer-events", "none")
|
||||
.style("background-color", "white")
|
||||
.style("border", "solid")
|
||||
.style("border-width", "2px")
|
||||
|
|
@ -44,24 +46,28 @@ const create_heatmap_week_hour = (container_id, data, options) => {
|
|||
.style("padding", "5px")
|
||||
|
||||
// Three function that change the tooltip when user hover / move / leave a cell
|
||||
const mouseover = function(d) {
|
||||
tooltip.style("opacity", 1)
|
||||
d3.select(this)
|
||||
const mouseover = function(event, d) {
|
||||
d3.select(event.target)
|
||||
.style("stroke", "black")
|
||||
//.style("stroke-opacity", 1)
|
||||
.style("stroke-opacity", 1)
|
||||
|
||||
var xPosition = d3.mouse(this)[0] + margin.left;
|
||||
var yPosition = d3.mouse(this)[1] + margin.top + window.scrollY + 100;
|
||||
let d3_pageX = event.pageX;
|
||||
let d3_pageY = event.pageY;
|
||||
|
||||
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
|
||||
.style("left", xPosition + "px")
|
||||
.style("top", yPosition + "px");
|
||||
.style("left", (d3_pageX) + "px")
|
||||
.style("top", (d3_pageY - 28) + "px");
|
||||
tooltip.transition()
|
||||
.duration(200)
|
||||
.style("opacity", 1);
|
||||
}
|
||||
const mouseleave = function(d) {
|
||||
tooltip.style("opacity", 0)
|
||||
d3.select(this)
|
||||
const mouseleave = function(event, d) {
|
||||
d3.select(event.target)
|
||||
.style("stroke", "white")
|
||||
//.style("stroke-opacity", 0.8)
|
||||
tooltip.transition()
|
||||
.duration(200)
|
||||
.style("opacity", 0)
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -114,22 +120,22 @@ const create_heatmap_week_hour = (container_id, data, options) => {
|
|||
var heatMap = svg.selectAll(".hour")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("x", function (d) {
|
||||
return (d.hour - 1) * gridSize;
|
||||
})
|
||||
.attr("y", function (d) {
|
||||
return (d.day - 1) * gridSize;
|
||||
})
|
||||
.attr("class", "hour bordered")
|
||||
.attr("width", gridSize)
|
||||
.attr("height", gridSize)
|
||||
.style("stroke", "white")
|
||||
.style("stroke-opacity", 0.6)
|
||||
.style("fill", function (d) {
|
||||
return colorScale(d.count);
|
||||
})
|
||||
.on("mouseover", mouseover)
|
||||
.on("mouseleave", mouseleave);
|
||||
.attr("x", function (d) {
|
||||
return (d.hour - 1) * gridSize;
|
||||
})
|
||||
.attr("y", function (d) {
|
||||
return (d.day - 1) * gridSize;
|
||||
})
|
||||
.attr("class", "hour bordered")
|
||||
.attr("width", gridSize)
|
||||
.attr("height", gridSize)
|
||||
.style("stroke", "white")
|
||||
.style("stroke-opacity", 0.6)
|
||||
.style("fill", function (d) {
|
||||
return colorScale(d.count);
|
||||
})
|
||||
.on("mouseover", mouseover)
|
||||
.on("mouseleave", mouseleave);
|
||||
|
||||
//Append title to the top
|
||||
svg.append("text")
|
||||
|
|
@ -221,8 +227,7 @@ const create_heatmap_week_hour = (container_id, data, options) => {
|
|||
.attr("class", "axis")
|
||||
.attr("transform", "translate(0," + (10) + ")")
|
||||
.call(xAxis);
|
||||
|
||||
|
||||
|
||||
// return svg ???
|
||||
// return svg
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// sanitise str_to_sanitize
|
||||
function sanitize_text(str_to_sanitize) {
|
||||
return $("<span\>").text(str_to_sanitize).html()
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@
|
|||
<i class="fas fa-user-circle"></i>
|
||||
<i class="far fa-comment-dots"></i>
|
||||
</span>
|
||||
{{ meta["nb_messages"] }}
|
||||
<a class="badge-primary px-1 py-0" href="{{ url_for('chats_explorer.objects_user_account_chat') }}?subtype={{ meta['subtype'] }}&id={{ main_obj_id }}&chat_id={{ meta['id'] }}">{{ meta["nb_messages"] }} </a>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -95,6 +95,10 @@
|
|||
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
|
||||
</button>
|
||||
</a>
|
||||
<a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
|
||||
<button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships
|
||||
</button>
|
||||
</a>
|
||||
{% endif %}
|
||||
</span>
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,11 @@
|
|||
{% else %}
|
||||
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
|
||||
<button class="btn btn-info"><i class="far fa-eye"></i> Correlations
|
||||
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
|
||||
</button>
|
||||
</a>
|
||||
<a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
|
||||
<button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships
|
||||
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
|
||||
</button>
|
||||
</a>
|
||||
|
|
|
|||
|
|
@ -12,12 +12,14 @@
|
|||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
|
||||
|
||||
|
||||
</head>
|
||||
|
|
@ -34,6 +36,9 @@
|
|||
|
||||
{% include 'chats_explorer/card_user_account.html' %}
|
||||
|
||||
<span class="mt-3">
|
||||
{% include 'objects/image/block_blur_img_slider.html' %}
|
||||
</span>
|
||||
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
|
||||
{% include 'chats_explorer/block_translation.html' %}
|
||||
{% endwith %}
|
||||
|
|
@ -42,10 +47,16 @@
|
|||
<h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>
|
||||
<div id="heatmapweekhourall"></div>
|
||||
|
||||
<h4>Numbers of Messages Posted by Chat:</h4>
|
||||
<div id="chord_user_chats" style="max-width: 900px"></div>
|
||||
|
||||
<h4>Numbers of Mentions:</h4>
|
||||
<div id="chord_mentions" style="max-width: 900px"></div>
|
||||
|
||||
<h4>User Chats:</h4>
|
||||
{% for meta_chats in meta['chats'] %}
|
||||
<div class="my-2">
|
||||
{% with meta=meta_chats %}
|
||||
{% with meta=meta_chats,main_obj_id=meta['id'] %}
|
||||
{% include 'chats_explorer/basic_card_chat.html' %}
|
||||
{% endwith %}
|
||||
</div>
|
||||
|
|
@ -58,6 +69,8 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
{% include 'objects/tooltip_ail_objects.html' %}
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
|
|
@ -77,26 +90,19 @@ d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?su
|
|||
create_heatmap_week_hour('#heatmapweekhourall', data);
|
||||
})
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
let url = "{{ url_for('chats_explorer.objects_user_account_chats_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
|
||||
d3.json(url).then(function(data) {
|
||||
create_directed_chord_diagram('#chord_user_chats', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
|
||||
});
|
||||
|
||||
let url2 = "{{ url_for('chats_explorer.objects_user_account_mentions_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
|
||||
d3.json(url2).then(function(data) {
|
||||
create_directed_chord_diagram('#chord_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
|
||||
});
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
|
|||
133
var/www/templates/chats_explorer/user_chat_messages.html
Normal file
133
var/www/templates/chats_explorer/user_chat_messages.html
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>User Account - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<h3>User:</h3>
|
||||
{% with meta=meta['user-account'] %}
|
||||
{% include 'chats_explorer/card_user_account.html' %}
|
||||
{% endwith %}
|
||||
|
||||
<h3>Chat:</h3>
|
||||
{% with meta=meta['chat'] %}
|
||||
{% include 'chats_explorer/basic_card_chat.html' %}
|
||||
{% endwith %}
|
||||
|
||||
<div class="mt-2">
|
||||
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
|
||||
{% include 'chats_explorer/block_translation.html' %}
|
||||
{% endwith %}
|
||||
{% include 'objects/image/block_blur_img_slider.html' %}
|
||||
</div>
|
||||
|
||||
|
||||
<div class="position-relative">
|
||||
<div class="chat-messages p-2">
|
||||
|
||||
{% for date in meta['messages'] %}
|
||||
|
||||
<div class="divider d-flex align-items-center mb-4">
|
||||
<p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
|
||||
<span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{% for mess in meta['messages'][date] %}
|
||||
|
||||
{% with message=mess %}
|
||||
{% include 'chats_explorer/block_message.html' %}
|
||||
{% endwith %}
|
||||
|
||||
{% endfor %}
|
||||
<br>
|
||||
{% endfor %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{# {% if meta['chats'] %}#}
|
||||
{# <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>#}
|
||||
{# <div id="heatmapweekhourall"></div>#}
|
||||
{##}
|
||||
{# <h4>User Chats:</h4>#}
|
||||
{# {% for meta_chats in meta['chats'] %}#}
|
||||
{# <div class="my-2">#}
|
||||
{# {% with meta=meta_chats %}#}
|
||||
{# {% include 'chats_explorer/basic_card_chat.html' %}#}
|
||||
{# {% endwith %}#}
|
||||
{# </div>#}
|
||||
{# {% endfor %}#}
|
||||
{# {% endif %}#}
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_chat").addClass("active");
|
||||
|
||||
});
|
||||
|
||||
{#d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}")#}
|
||||
{# .then(function(data) {#}
|
||||
{# create_heatmap_week_hour('#heatmapweekhourall', data);#}
|
||||
{# })#}
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -16,7 +16,8 @@
|
|||
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.icon_legend {
|
||||
|
|
@ -71,6 +72,7 @@
|
|||
|
||||
.blured {
|
||||
filter: blur(5px);
|
||||
max-width: 200px;
|
||||
}
|
||||
|
||||
.graph_panel {
|
||||
|
|
@ -127,6 +129,10 @@
|
|||
{% include 'correlation/metadata_card_item.html' %}
|
||||
{% endif %}
|
||||
|
||||
<div class="my-2">
|
||||
{% include 'objects/image/block_blur_img_slider.html' %}
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-10">
|
||||
|
||||
|
|
@ -205,13 +211,49 @@
|
|||
<div class="card-body text-center px-0 py-0">
|
||||
|
||||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info">Relationship</li>
|
||||
<li class="list-group-item list-group-item-info">Relationships</li>
|
||||
<form action="{{ url_for('correlation.show_relationship') }}" method="post">
|
||||
|
||||
<li class="list-group-item text-left">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_from_Check" name="relationship_forwarded_from_Check" {%if "forwarded_from" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_forwarded_from_Check">Forwarded From</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_to_Check" name="relationship_forwarded_to_Check" {%if "forwarded_to" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_forwarded_to_Check">Forwarded To</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_in_Check" name="relationship_in_Check" {%if "in" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_in_Check">In</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_mention_Check">Mention</label>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
|
||||
<input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
|
||||
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
|
||||
|
||||
<li class="list-group-item list-group-item-info">Objects Types</li>
|
||||
|
||||
<li class="list-group-item text-left">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="chat_Check">Chat</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="message_Check">Message</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="user-account_Check">User-Account</label>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
{# <li class="list-group-item text-left">#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
|
||||
|
|
@ -313,10 +355,25 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<h3>Forwards</h3>
|
||||
<div id="chart_test" style="max-width: 900px"></div>
|
||||
|
||||
<h3>Mentions</h3>
|
||||
<div id="chart_mentions" style="max-width: 900px"></div>
|
||||
|
||||
|
||||
<table id="table_graph_node_objects">
|
||||
</table>
|
||||
|
||||
<div id="timeline"></div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
{% include 'objects/tooltip_ail_objects.html' %}
|
||||
<script>
|
||||
|
||||
var all_graph = {};
|
||||
|
|
@ -324,8 +381,20 @@ $(document).ready(function(){
|
|||
$("#incomplete_graph").hide();
|
||||
$("#page-Decoded").addClass("active");
|
||||
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||
all_graph.onResize();
|
||||
|
||||
let url = "{{ url_for('correlation.relationships_chord_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
|
||||
d3.json(url).then(function(data) {
|
||||
create_directed_chord_diagram('#chart_test', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
|
||||
});
|
||||
|
||||
let url2 = "{{ url_for('correlation.relationships_chord_mentions_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
|
||||
d3.json(url2).then(function(data) {
|
||||
create_directed_chord_diagram('#chart_mentions', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
const blur_slider_correlation = $('#blur-slider-correlation');
|
||||
|
|
@ -553,30 +622,41 @@ d3.json(url)
|
|||
|
||||
d3.select("body").on("keypress", keypressed)
|
||||
|
||||
let table_obj = document.getElementById("table_graph_node_objects")
|
||||
for (let i=0; i<data.nodes.length; i++) {
|
||||
let newRow = table_obj.insertRow();
|
||||
let newCell = newRow.insertCell();
|
||||
let newText = document.createTextNode(data.nodes[i].id);
|
||||
newCell.appendChild(newText);
|
||||
|
||||
//console.log(data.nodes[i])
|
||||
|
||||
}
|
||||
|
||||
//// --------------------------------------------------------------------------------------------------------
|
||||
|
||||
})
|
||||
.catch(function(error) {
|
||||
$("#graph_loading").remove()
|
||||
svg_node.remove();
|
||||
d3.select("#graph").append("div")
|
||||
.text(error);
|
||||
.text(error);
|
||||
});
|
||||
}
|
||||
|
||||
function zoomed() {
|
||||
container_graph.attr("transform", d3.event.transform);
|
||||
function zoomed(event, d) {
|
||||
container_graph.attr("transform", event.transform);
|
||||
}
|
||||
|
||||
function doubleclick (d) {
|
||||
function doubleclick (event, d) {
|
||||
window.open(d.url, '_blank');
|
||||
}
|
||||
|
||||
function keypressed () {
|
||||
//console.log(d3.event.keyCode)
|
||||
function keypressed (event, d) {
|
||||
//console.log(event.keyCode)
|
||||
//console.log(currentObject.id)
|
||||
// hide node, H or h key
|
||||
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
|
||||
if ((event.keyCode === 72 || event.keyCode === 104) && currentObject) {
|
||||
window.location.href = correl_link + "&hide=" + currentObject.id
|
||||
}
|
||||
|
||||
|
|
@ -586,29 +666,29 @@ function click (d) {
|
|||
console.log('clicked')
|
||||
}
|
||||
|
||||
function drag_start(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
|
||||
function drag_start(event, d) {
|
||||
if (!event.active) simulation.alphaTarget(0.3).restart();
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function dragged(d) {
|
||||
d.fx = d3.event.x;
|
||||
d.fy = d3.event.y;
|
||||
function dragged(event, d) {
|
||||
d.fx = event.x;
|
||||
d.fy = event.y;
|
||||
}
|
||||
|
||||
function drag_end(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0);
|
||||
function drag_end(event, d) {
|
||||
if (!event.active) simulation.alphaTarget(0);
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function mouseovered(d) {
|
||||
function mouseovered(event, d, obj_gid, obj_label, additional_text) {
|
||||
|
||||
currentObject = d;
|
||||
|
||||
var d3_pageX = d3.event.pageX;
|
||||
var d3_pageY = d3.event.pageY;
|
||||
var d3_pageX = event.pageX;
|
||||
var d3_pageY = event.pageY;
|
||||
|
||||
if (d.popover) {
|
||||
div.html(d.popover)
|
||||
|
|
@ -621,8 +701,16 @@ if (d.popover) {
|
|||
blur_tooltip();
|
||||
} else {
|
||||
|
||||
var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
|
||||
var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
|
||||
let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
|
||||
if (obj_label) {
|
||||
pop_header = pop_header + sanitize_text(obj_label)
|
||||
} else if (obj_gid) {
|
||||
pop_header = pop_header + sanitize_text(obj_gid)
|
||||
} else {
|
||||
pop_header = pop_header + sanitize_text(d.text)
|
||||
}
|
||||
pop_header = pop_header + "</h6></div>"
|
||||
let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
|
||||
|
||||
div.html(pop_header + spinner)
|
||||
.style("left", (d3_pageX) + "px")
|
||||
|
|
@ -632,9 +720,16 @@ if (d.popover) {
|
|||
.duration(200)
|
||||
.style("opacity", 1);
|
||||
|
||||
$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
|
||||
let description_url = "{{ url_for('correlation.get_description') }}?object_id="
|
||||
if (obj_gid) {
|
||||
description_url = description_url + obj_gid
|
||||
} else {
|
||||
description_url = description_url + d.id
|
||||
}
|
||||
|
||||
$.getJSON(description_url,
|
||||
function(data){
|
||||
var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
|
||||
let desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
|
||||
Object.keys(data).forEach(function(key) {
|
||||
if (key=="status") {
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
|
||||
|
|
@ -650,9 +745,16 @@ if (d.popover) {
|
|||
desc = desc + "fa-times-circle\"></i>DOWN"
|
||||
}
|
||||
desc = desc + "</div></dd>"
|
||||
} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
|
||||
}
|
||||
} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
|
||||
if (data[key]) {
|
||||
if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
|
||||
let date = sanitize_text(data[key])
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
|
||||
} else {
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
desc = desc + "</dl>"
|
||||
|
||||
|
|
@ -674,6 +776,9 @@ if (d.popover) {
|
|||
desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
|
||||
}
|
||||
}
|
||||
if (additional_text) {
|
||||
desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
|
||||
}
|
||||
|
||||
desc = desc + "</div></div>"
|
||||
div.html(desc)
|
||||
|
|
@ -697,7 +802,6 @@ if (d.popover) {
|
|||
|
||||
function mouseouted() {
|
||||
currentObject = null;
|
||||
|
||||
div.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
|
|
@ -712,8 +816,18 @@ all_graph.onResize = function () {
|
|||
}
|
||||
|
||||
window.all_graph = all_graph;
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/*
|
||||
*/
|
||||
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -95,6 +95,10 @@
|
|||
<div id="barchart_type_month"></div>
|
||||
<div class="text-center" id="pie_chart_month"></div>
|
||||
|
||||
<h3>Previous Month Stats:</h3>
|
||||
<div id="barchart_type_month"></div>
|
||||
<div class="text-center" id="barchart_type_previous_month"></div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
@ -156,6 +160,13 @@ $(document).ready(function(){
|
|||
}
|
||||
);
|
||||
|
||||
$.getJSON("{{ url_for('crawler_splash.crawlers_last_domains_previous_month_json') }}?type={{type}}",
|
||||
function (data) {
|
||||
let div_width = $("#barchart_type_previous_month").width();
|
||||
barchart_stack("barchart_type_previous_month", data, {"width": div_width});
|
||||
}
|
||||
);
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@
|
|||
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" {% if not dict_tracker['filters'] or 'decoded' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i> Decoded <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Content that has been decoded from an encoded format, such as base64"></i></label>
|
||||
</div>
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
|
|
@ -83,7 +83,7 @@
|
|||
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i> Domain</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" {% if not dict_tracker['filters'] or 'item' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i> Item <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Text that has been processed by AIL. It can include various types of extracted information"></i></label>
|
||||
</div>
|
||||
<div class="card border-dark mb-4" id="sources_item_div">
|
||||
|
|
@ -100,14 +100,14 @@
|
|||
|
||||
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" {% if not dict_tracker['filters'] or 'pgp' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i> PGP <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="PGP key/block metadata"></i></label>
|
||||
</div>
|
||||
<div class="card border-dark mb-4" id="sources_pgp_div">
|
||||
<div class="card-body">
|
||||
<h6>Filter PGP by subtype:</h6>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'name' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
|
||||
<label class="custom-control-label" for="filter_pgp_name">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
|
|
@ -119,7 +119,7 @@
|
|||
</label>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'mail' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
|
||||
<label class="custom-control-label" for="filter_pgp_mail">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
|
|
@ -133,15 +133,15 @@
|
|||
</div>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" {% if not dict_tracker['filters'] or 'title' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="title_obj"><i class="fas fa-heading"></i> Title <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Title that has been extracted from a HTML page"></i></label>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" {% if not dict_tracker['filters'] or 'message' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i> Message <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Messages from Chats"></i></label>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" checked="">
|
||||
<input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" {% if not dict_tracker['filters'] or 'ocr' in dict_tracker['filters'] %}checked=""{% endif %}>
|
||||
<label class="custom-control-label" for="ocr_obj"><i class="fas fa-comment-dots"></i> OCR <i class="fas fa-expand text-info" data-toggle="tooltip" data-placement="right" title="Text extracted from Images"></i></label>
|
||||
</div>
|
||||
|
||||
|
|
@ -310,6 +310,9 @@ $(document).ready(function(){
|
|||
emptyText: 'Item Sources to track (ALL IF EMPTY)',
|
||||
});
|
||||
|
||||
item_source_input_controller();
|
||||
pgp_source_input_controller();
|
||||
|
||||
$('#tracker_type').on('change', function() {
|
||||
var tracker_type = this.value;
|
||||
if (tracker_type=="word") {
|
||||
|
|
|
|||
136
var/www/templates/objects/tooltip_ail_objects.html
Normal file
136
var/www/templates/objects/tooltip_ail_objects.html
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
<script>
|
||||
let tooltip_ail_obj = d3.select("body").append("div")
|
||||
//.attr("class", "tooltip_graph")
|
||||
.attr("id", "tooltip_graph")
|
||||
.style("opacity", 0)
|
||||
.style("position", "absolute")
|
||||
.style("text-align", "center")
|
||||
.style("padding", "2px")
|
||||
.style("font", "12px sans-serif")
|
||||
.style("background", "#ebf4fb")
|
||||
.style("border", "2px solid #b7ddf2")
|
||||
.style("border-radius", "8px")
|
||||
.style("pointer-events", "none")
|
||||
.style("color", "#000000");
|
||||
|
||||
function mouseover_tooltip_ail_obj(event, d, obj_gid, obj_label, additional_text) { /// div var/const tooltip tooltip_ail_obj
|
||||
|
||||
let d3_pageX = event.pageX;
|
||||
let d3_pageY = event.pageY;
|
||||
|
||||
if (d.popover) {
|
||||
tooltip_ail_obj.html(d.popover)
|
||||
.style("left", (d3_pageX) + "px")
|
||||
.style("top", (d3_pageY - 28) + "px");
|
||||
|
||||
tooltip_ail_obj.transition()
|
||||
.duration(200)
|
||||
.style("opacity", 1);
|
||||
blur_images();
|
||||
} else {
|
||||
|
||||
let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
|
||||
if (obj_label) {
|
||||
pop_header = pop_header + sanitize_text(obj_label)
|
||||
} else if (obj_gid) {
|
||||
pop_header = pop_header + sanitize_text(obj_gid)
|
||||
} else {
|
||||
pop_header = pop_header + sanitize_text(d.text)
|
||||
}
|
||||
pop_header = pop_header + "</h6></div>"
|
||||
let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
|
||||
|
||||
tooltip_ail_obj.html(pop_header + spinner)
|
||||
.style("left", (d3_pageX) + "px")
|
||||
.style("top", (d3_pageY - 28) + "px");
|
||||
|
||||
tooltip_ail_obj.transition()
|
||||
.duration(200)
|
||||
.style("opacity", 1);
|
||||
|
||||
let description_url = "{{ url_for('correlation.get_description') }}?object_id="
|
||||
if (obj_gid) {
|
||||
description_url = description_url + obj_gid
|
||||
} else {
|
||||
description_url = description_url + d.id
|
||||
}
|
||||
|
||||
let desc
|
||||
$.getJSON(description_url, function(data){
|
||||
desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
|
||||
Object.keys(data).forEach(function(key) {
|
||||
if (key==="status") {
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
|
||||
if (data["status"]) {
|
||||
desc = desc + "Green"
|
||||
} else {
|
||||
desc = desc + "Red"
|
||||
}
|
||||
desc = desc + ";\"><i class=\"fas "
|
||||
if (data["status"]) {
|
||||
desc = desc + "fa-check-circle\"></i>UP"
|
||||
} else {
|
||||
desc = desc + "fa-times-circle\"></i>DOWN"
|
||||
}
|
||||
desc = desc + "</div></dd>"
|
||||
} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
|
||||
if (data[key]) {
|
||||
if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
|
||||
let date = sanitize_text(data[key])
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
|
||||
} else {
|
||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
desc = desc + "</dl>"
|
||||
|
||||
if (data["tags"]) {
|
||||
data["tags"].forEach(function(tag) {
|
||||
desc = desc + "<span class=\"badge badge-warning\">"+ sanitize_text(tag) +"</span>";
|
||||
});
|
||||
}
|
||||
|
||||
if (data["img"]) {
|
||||
if (data["tags_safe"]) {
|
||||
if (data["type"] === "screenshot") {
|
||||
desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
|
||||
} else {
|
||||
desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
|
||||
}
|
||||
desc = desc + data["img"] +" class=\"object_image\" id=\"tooltip_screenshot_correlation\" style=\"filter: blur(5px);max-width: 200px;\"/>";
|
||||
} else {
|
||||
desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
|
||||
}
|
||||
}
|
||||
if (additional_text) {
|
||||
desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
|
||||
}
|
||||
|
||||
desc = desc + "</div></div>"
|
||||
tooltip_ail_obj.html(desc)
|
||||
.style("left", (d3_pageX) + "px")
|
||||
.style("top", (d3_pageY - 28) + "px");
|
||||
d.popover = desc
|
||||
|
||||
if (data["img"]) {
|
||||
blur_images();
|
||||
}
|
||||
|
||||
})
|
||||
.fail(function(error) {
|
||||
desc = pop_header + "<div class=\"card-body bg-dark pt-0\"><i class=\"fas fa-3x fa-times text-danger\"></i>"+ error.statusText +"</div>"
|
||||
tooltip_ail_obj.html(desc)
|
||||
.style("left", (d3_pageX) + "px")
|
||||
.style("top", (d3_pageY - 28) + "px");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function mouseout_tooltip_ail_obj(event, d) {
|
||||
tooltip_ail_obj.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
}
|
||||
</script>
|
||||
|
|
@ -9,7 +9,9 @@ wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.
|
|||
|
||||
BOOTSTRAP_VERSION='4.2.1'
|
||||
FONT_AWESOME_VERSION='5.7.1'
|
||||
|
||||
D3_JS_VERSION='5.16.0'
|
||||
wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
|
||||
|
||||
rm -rf temp
|
||||
mkdir temp
|
||||
|
|
|
|||
Loading…
Reference in a new issue