Merge branch 'relationships' into otp

This commit is contained in:
terrtia 2024-06-25 09:47:38 +02:00
commit 8d4721d703
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
30 changed files with 1368 additions and 233 deletions

View file

@ -44,6 +44,11 @@ class DefaultFeeder:
def get_date(self):
return datetime.date.today().strftime("%Y%m%d")
def get_feeder_timestamp(self):
timestamp = self.json_data.get('timestamp')
if timestamp:
return int(timestamp)
def get_json_data(self):
"""
Return the JSON data,

View file

@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
import datetime
import os
import sys
import time
from abc import ABC
@ -127,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta'].get('reply_to', {}).get('message_id')
def get_message_forward(self):
return self.json_data['meta'].get('forward')
return self.json_data['meta'].get('forward', {})
def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data'])
@ -164,7 +165,40 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
self.obj = Messages.Message(obj_id)
return self.obj
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
# TODO handle subchannel
def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
# Obj Daterange
chat.add(date)
if meta_chat.get('name'):
chat.set_name(meta_chat['name'])
if meta_chat.get('info'):
chat.set_info(meta_chat['info'])
if meta_chat.get('date'): # TODO check if already exists
chat.set_created_at(int(meta_chat['date']['timestamp']))
if meta_chat.get('icon'):
img = Images.create(meta_chat['icon'], b64=True)
img.add(date, chat)
chat.set_icon(img.get_global_id())
if new_objs:
new_objs.add(img)
if meta_chat.get('username'):
username = Username(meta_chat['username'], self.get_chat_protocol())
chat.update_username_timeline(username.get_global_id(), int(time.time()))
username.add(date, obj=chat) # TODO TODAY DATE
return chat
##############################################################################################################################
def process_chat(self, new_objs, obj, date, timestamp, feeder_timestamp, reply_id=None):
meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
subchannel = None
@ -190,7 +224,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
chat.update_username_timeline(username.get_global_id(), timestamp)
chat.update_username_timeline(username.get_global_id(), feeder_timestamp)
if meta.get('subchannel'):
subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
@ -257,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# else:
# # ADD NEW MESSAGE REF (used by discord)
##########################################################################################
def _process_user(self, meta, date, timestamp, new_objs=None):
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
# TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message
username.add(date) # TODO # correlation message ??? ###############################################################
# ADDITIONAL METAS
if meta.get('firstname'):
user_account.set_first_name(meta['firstname'])
if meta.get('lastname'):
user_account.set_last_name(meta['lastname'])
if meta.get('phone'):
user_account.set_phone(meta['phone'])
if meta.get('icon'):
img = Images.create(meta['icon'], b64=True)
img.add(date, user_account)
user_account.set_icon(img.get_global_id())
new_objs.add(img)
if meta.get('info'):
user_account.set_info(meta['info'])
user_account.add(date)
return user_account
def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta'].get('sender')
if not meta:
@ -305,8 +372,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if self.obj:
objs.add(self.obj)
new_objs = set()
chats_objs = set()
date, timestamp = self.get_message_date_timestamp()
feeder_timestamp = self.get_feeder_timestamp()
if not feeder_timestamp:
feeder_timestamp = timestamp
# REPLY
reply_id = self.get_message_reply_id()
@ -363,81 +434,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
print(obj.id)
# CHAT
chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
# # TODO HANDLE OTHERS OBJECT TYPE
# # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# # Message forward + Discussion
# if self.get_json_meta().get('forward'):
# discussion_id = self.get_json_meta().get('discussion')
# forward_from = self.get_message_forward()
#
# if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
# chat_forward_id = forward_from['from']['id']
# message_forward_id = forward_from['from']['channel_post']
#
# # if chat_forward_id == discussion_id:
# # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
# # if linked_chat.exists():
# # # create thread
# # # add message replies for each childrens
#
# # TODO HANDLE THREAD
# # TODO Change FORWARD META FIELDS
# # meta['forward'] = {}
# # # CHAT ID
# # # SUBCHANNEL ID -> can be None
# # # Message ID
#
# # meta['forward']['origin']
# # # same as 'forward'
#
# if self.get_json_meta().get('forward'):
# forward = self.get_message_forward()
# f_chat = forward['chat']
# f_subchannel = forward.get('subchannel')
# f_id = forward.get('id')
# if not f_subchannel:
# chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
# if chat_forward.exists():
# for chat_obj in chat_objs:
# if chat_obj.type == 'chat':
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
# # TODO LIST FORWARDED MESSAGES
#
#
# # Discord -> serverID + subchannel ID + message ID
# # Telegram -> chat ID + Message ID
# # + ORIGIN IDs
#
#
#
# # TODO create relationships graph
#
#
# # TODO REMOVE ME
# # Message forward # TODO handle subchannel + message ID
# if self.get_json_meta().get('forward'):
# forward_from = self.get_message_forward()
# print('-----------------------------------------------------------')
# print(forward_from)
# if forward_from:
# forward_from_type = forward_from['from']['type']
# if forward_from_type == 'channel' or forward_from_type == 'chat':
# chat_forward_id = forward_from['from']['id']
# chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
# if chat_forward.exists():
# for chat_obj in chat_objs:
# if chat_obj.type == 'chat':
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
# # chat_forward.add_relationship(obj.get_global_id(), 'forward')
curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, feeder_timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(new_objs, obj, date, timestamp)
user_account = self.process_sender(new_objs, obj, date, feeder_timestamp)
if user_account:
# UserAccount---ChatObjects
for obj_chat in chat_objs:
for obj_chat in curr_chats_objs:
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
# if chat: # TODO Chat---Username correlation ???
@ -449,4 +453,73 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# -> subchannel ?
# -> thread id ?
chats_objs.update(curr_chats_objs)
#######################################################################
## FORWARD ##
chat_fwd = None
user_fwd = None
if self.get_json_meta().get('forward'):
meta_fwd = self.get_message_forward()
if meta_fwd.get('chat'):
chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
if meta_fwd.get('user'):
user_fwd = self._process_user(meta_fwd['user'], date, feeder_timestamp, new_objs=new_objs) # TODO date, timestamp ???
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
# TODO chat_fwd -> message
if chat_fwd or user_fwd:
for obj in objs:
if obj.type == 'message':
if chat_fwd:
obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
if user_fwd:
obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in')
# -FORWARD- #
## MENTION ##
if self.get_json_meta().get('mentions'):
for mention in self.get_json_meta()['mentions'].get('chats', []):
m_obj = self._process_chat(mention, date, new_objs=new_objs)
if m_obj:
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
# TODO PERF
# TODO Keep message obj + chat obj in global var
for obj in objs:
if obj.type == 'message':
obj.add_relationship(m_obj.get_global_id(), 'mention')
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in')
for mention in self.get_json_meta()['mentions'].get('users', []):
m_obj = self._process_user(mention, date, feeder_timestamp, new_objs=new_objs) # TODO date ???
if m_obj:
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
# TODO PERF
# TODO Keep message obj + chat obj in global var
for obj in objs:
if obj.type == 'message':
obj.add_relationship(m_obj.get_global_id(), 'mention')
for chat_obj in chats_objs:
if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in')
# -MENTION- #
return new_objs | objs

View file

@ -762,6 +762,9 @@ def delete_obj_trackers(obj_type, subtype, obj_id):
#### TRACKERS ACL ####
## LEVEL ##
def is_tracker_global_level(tracker_uuid):
return int(r_tracker.hget(f'tracker:{tracker_uuid}', 'level')) == 1
def is_tracked_in_global_level(tracked, tracker_type):
for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
tracker = Tracker(tracker_uuid)
@ -805,6 +808,19 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
return {"status": "error", "reason": "Access Denied"}, 403
return {"uuid": tracker_uuid}, 200
def api_is_allowed_to_access_tracker(tracker_uuid, user_id):
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
if not tracker_creator:
return {"status": "error", "reason": "Unknown uuid"}, 404
user = User(user_id)
if not is_tracker_global_level(tracker_uuid):
if not user.is_in_role('admin') and user_id != tracker_creator:
return {"status": "error", "reason": "Access Denied"}, 403
return {"uuid": tracker_uuid}, 200
##-- ACL --##
#### FIX DB #### TODO ###################################################################

View file

@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
if obj_type == 'chat-thread':
return r_object.smembers(f'all_chat-thread:subtypes')
if obj_type == 'cryptocurrency':
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
if obj_type == 'pgp':
return ['key', 'mail', 'name']
if obj_type == 'username':

View file

@ -328,6 +328,31 @@ def get_username_meta_from_global_id(username_global_id):
username = Usernames.Username(username_id, instance_uuid)
return username.get_meta(options={'icon'})
###############################################################################
# TODO Pagination
def list_messages_to_dict(l_messages_id, translation_target=None):
options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
meta = {}
curr_date = None
for mess_id in l_messages_id:
message = Messages.Message(mess_id[1:])
timestamp = message.get_timestamp()
date_day = message.get_date()
date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}'
if date_day != curr_date:
meta[date_day] = []
curr_date = date_day
meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
meta[date_day].append(meta_mess)
# if mess_dict.get('tags'):
# for tag in mess_dict['tags']:
# if tag not in tags:
# tags[tag] = 0
# tags[tag] += 1
# return messages, pagination, tags
return meta
# TODO Filter
## Instance type
## Chats IDS
@ -354,7 +379,7 @@ def get_messages_iterator(filters={}):
# threads
for threads in chat.get_threads():
thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
_, _ = thread._get_messages(nb=-1)
messages, _ = thread._get_messages(nb=-1)
for mess in messages:
message_id, _, message_id = mess[0].split(':', )
yield Messages.Message(message_id)
@ -404,6 +429,15 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
meta.append(chat_meta)
return meta
def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
meta = {}
chat = Chats.Chat(chat_id, subtype)
chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
if chat_meta['username']:
chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None)
return meta
def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
week = {}
@ -431,6 +465,60 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
nb_day += 1
return stats
def get_user_account_chats_chord(subtype, user_id):
nb = {}
user_account = UsersAccount.UserAccount(user_id, subtype)
for chat_g_id in user_account.get_chats():
c_subtype, c_id = chat_g_id.split(':', 1)
chat = Chats.Chat(c_id, c_subtype)
nb[f'chat:{chat_g_id}'] = len(chat.get_user_messages(user_id))
user_account_gid = user_account.get_global_id() # # #
chord = {'meta': {}, 'data': []}
label = get_chat_user_account_label(user_account_gid)
if label:
chord['meta'][user_account_gid] = label
else:
chord['meta'][user_account_gid] = user_account_gid
for chat_g_id in nb:
label = get_chat_user_account_label(chat_g_id)
if label:
chord['meta'][chat_g_id] = label
else:
chord['meta'][chat_g_id] = chat_g_id
chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
return chord
def get_user_account_mentions_chord(subtype, user_id):
chord = {'meta': {}, 'data': []}
nb = {}
user_account = UsersAccount.UserAccount(user_id, subtype)
user_account_gid = user_account.get_global_id()
label = get_chat_user_account_label(user_account_gid)
if label:
chord['meta'][user_account_gid] = label
else:
chord['meta'][user_account_gid] = user_account_gid
for mess in user_account.get_messages():
m = Messages.Message(mess[9:])
for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
if rel:
if not rel['target'] in nb:
nb[rel['target']] = 0
nb[rel['target']] += 1
for g_id in nb:
label = get_chat_user_account_label(g_id)
if label:
chord['meta'][g_id] = label
else:
chord['meta'][g_id] = g_id
chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
return chord
def _get_chat_card_meta_options():
return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
@ -481,18 +569,65 @@ def fix_correlations_subchannel_message():
#### API ####
def get_chat_user_account_label(chat_gid):
label = None
obj_type, subtype, obj_id = chat_gid.split(':', 2)
if obj_type == 'chat':
obj = get_obj_chat(obj_type, subtype, obj_id)
username = obj.get_username()
if username:
username = username.split(':', 2)[2]
name = obj.get_name()
if username and name:
label = f'{username} - {name}'
elif username:
label = username
elif name:
label = name
elif obj_type == 'user-account':
obj = UsersAccount.UserAccount(obj_id, subtype)
username = obj.get_username()
if username:
username = username.split(':', 2)[2]
name = obj.get_name()
if username and name:
label = f'{username} - {name}'
elif username:
label = username
elif name:
label = name
return label
def enrich_chat_relationships_labels(relationships):
meta = {}
for row in relationships:
if row['source'] not in meta:
label = get_chat_user_account_label(row['source'])
if label:
meta[row['source']] = label
else:
meta[row['source']] = row['source']
if row['target'] not in meta:
label = get_chat_user_account_label(row['target'])
if label:
meta[row['target']] = label
else:
meta[row['target']] = row['target']
return meta
def api_get_chat_service_instance(chat_instance_uuid):
chat_instance = ChatServiceInstance(chat_instance_uuid)
if not chat_instance.exists():
return {"status": "error", "reason": "Unknown uuid"}, 404
return chat_instance.get_meta({'chats'}), 200
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
# print(chat.get_obj_language_stats())
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target)
if meta['username']:
meta['username'] = get_username_meta_from_global_id(meta['username'])
if meta['subchannels']:
@ -500,7 +635,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
else:
if translation_target not in Language.get_translation_languages():
translation_target = None
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
if messages:
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id):
@ -602,6 +738,18 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
return meta, 200
def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None):
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
if not user_account.exists():
return {"status": "error", "reason": "Unknown user-account"}, 404
meta = get_user_account_chat_message(user_id, instance_uuid, chat_id)
meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False)
if resp[1] != 200:
return resp
meta['chat'] = resp[0]
return meta, 200
def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
if not user_account.exists():

View file

@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
return schedule_uuid
def _delete_schedules():
for schedule_uuid in get_schedulers_uuid():
schedule = CrawlerSchedule(schedule_uuid)
schedule.delete()
# TODO sanityze UUID
def api_delete_schedule(data):
schedule_uuid = data.get('uuid')
@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
external=external)
return task_uuid
## -- CRAWLER TASK -- ##
#### CRAWLER TASK API ####

View file

@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject):
pass
def is_valid_address(self):
if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin':
if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron':
return check_base58_address(self.id)
else:
return True
@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject):
return 'ZEC'
elif self.subtype == 'dash':
return 'DASH'
elif self.subtype == 'tron':
return 'TRX'
return None
def get_link(self, flask_context=False):
@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject):
def get_all_subtypes():
# return ail_core.get_object_all_subtypes(self.type)
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
# def build_crypto_regex(subtype, search_id):
@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol):
return 'zcash'
elif symbol == 'DASH':
return 'dash'
elif symbol == 'TRX':
return 'tron'
return None
@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype):
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == '':
pass
elif subtype == 'name':
pass
elif subtype == 'mail':
pass
return name_to_search
def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):

View file

@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
def get_last_name(self):
return self._get_field('lastname')
def get_name(self):
first_name = self.get_first_name()
last_name = self.get_last_name()
if first_name and last_name:
return f'{first_name} {last_name}'
elif first_name:
return first_name
elif last_name:
return last_name
def get_phone(self):
return self._get_field('phone')
@ -129,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
def get_messages(self):
messages = []
for mess in self.get_correlation('message'):
messages.append(f'message:{mess}')
correl = self.get_correlation('message')
if 'message' in correl:
for mess_id in correl['message']:
messages.append(f'message:{mess_id}')
return messages
def get_messages_by_chat_obj(self, chat_obj):
messages = []
for mess in self.get_correlation_iter_obj(chat_obj, 'message'):
@ -157,6 +168,7 @@ class UserAccount(AbstractSubtypeObject):
meta['usernames'] = self.get_usernames()
if 'icon' in options:
meta['icon'] = self.get_icon()
meta['svg_icon'] = meta['icon']
if 'info' in options:
meta['info'] = self.get_info()
if 'translation' in options and translation_target:

View file

@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
@ -299,6 +299,9 @@ class AbstractObject(ABC):
def get_nb_relationships(self, filter=[]):
return get_obj_nb_relationships(self.get_global_id())
def get_obj_relationships(self, relationships=set(), filter_types=set()):
return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
def add_relationship(self, obj2_global_id, relationship, source=True):
# is source
if source:

View file

@ -562,24 +562,47 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
# --- CORRELATION --- #
#### RELATIONSHIPS ####
def get_relationships():
return relationships_engine.get_relationships()
def sanitize_relationships(relationships):
return relationships_engine.sanitize_relationships(relationships)
def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_nb_relationships(filter=filter_types)
def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
objs_hidden=set(),
flask_context=False):
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
filter_types=filter_types,
max_nodes=max_nodes, level=level,
objs_hidden=objs_hidden)
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
filter_types=filter_types,
max_nodes=max_nodes, level=level,
objs_hidden=objs_hidden)
# print(meta)
meta['objs'] = list(meta['objs'])
return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
"links": links,
"meta": meta}
def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
if obj_type == 'chat':
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
data = relationships_engine.get_chat_forward_stats(obj_global_id)
return data
return []
def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
if obj_type == 'chat':
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
data = relationships_engine.get_chat_mentions_stats(obj_global_id)
return data
return []
# --- RELATIONSHIPS --- #
# if __name__ == '__main__':
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import json
import os
import sys
@ -16,41 +16,103 @@ config_loader = None
RELATIONSHIPS = {
"forward",
"forwarded_from",
"forwarded_to", # forwarded_to
"in",
"mention"
}
RELATIONSHIPS_OBJS = { # TODO forward user-account
"forwarded_from": {
'chat': {'message'},
'message': {'chat', 'user-account'}
},
"forwarded_to": {
'chat': {'chat'},
'user-account': {'chat'},
},
"in": {
'chat': {'message'},
'message': {'chat'}
},
"mention": {
'chat': {'chat', 'user-account', 'message'},
'message': {'chat', 'user-account'},
'user-account': {'chat', 'message'},
},
}
def get_relationships():
return RELATIONSHIPS
def get_obj_relationships_by_type(obj_global_id, relationship):
return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
def get_obj_nb_relationships_by_type(obj_global_id, relationship):
return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
def get_obj_relationships(obj_global_id):
relationships = []
for relationship in get_relationships():
for rel in get_obj_relationships_by_type(obj_global_id, relationship):
meta = {'relationship': relationship}
direction, obj_id = rel.split(':', 1)
if direction == 'i':
meta['source'] = obj_id
meta['target'] = obj_global_id
else:
meta['target'] = obj_id
meta['source'] = obj_global_id
if not obj_id.startswith('chat'):
continue
meta['id'] = obj_id
# meta['direction'] = direction
relationships.append(meta)
def sanitize_relationships(relationships):
rels = get_relationships()
if relationships:
relationships = set(relationships).intersection(rels)
if not relationships:
relationships = rels
if not relationships:
return []
return relationships
def get_obj_nb_relationships(obj_global_id):
def get_relationship_obj_types(relationship):
return RELATIONSHIPS_OBJS.get(relationship, {})
def get_relationship_objs(relationship, obj_type):
return get_relationship_obj_types(relationship).get(obj_type, set())
def sanityze_obj_types(relationship, obj_type, filter_types):
objs_types = get_relationship_objs(relationship, obj_type)
if filter_types:
filter_types = objs_types.intersection(filter_types)
else:
filter_types = objs_types
# if not filter_types:
# filter_types = objs_types
# if not filter_types:
# return []
return filter_types
# TODO check obj_type
# TODO sanitize relationships
def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
obj_type = obj_global_id.split(':', 1)[0]
relationships = {}
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
for o_type in filter_types:
relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
return relationships
def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
obj_type = obj_global_id.split(':', 1)[0]
relationships = {}
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
for o_type in filter_types:
relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
return relationships
def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
all_relationships = []
for relationship in relationships:
obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
for obj_type in obj_relationships:
for rel in obj_relationships[obj_type]:
meta = {'relationship': relationship}
direction, obj_id = rel.split(':', 1)
if direction == 'i':
meta['source'] = obj_id
meta['target'] = obj_global_id
else:
meta['target'] = obj_id
meta['source'] = obj_global_id
meta['id'] = obj_id
# meta['direction'] = direction
all_relationships.append(meta)
return all_relationships
def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
nb = {}
for relationship in get_relationships():
nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
@ -59,27 +121,27 @@ def get_obj_nb_relationships(obj_global_id):
# TODO Filter by obj type ???
def add_obj_relationship(source, target, relationship):
r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
# r_rel.sadd(f'rels:{source}', relationship)
# r_rel.sadd(f'rels:{target}', relationship)
source_type = source.split(':', 1)[0]
target_type = target.split(':', 1)[0]
r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
links = []
nodes = set()
meta = {'complete': True, 'objs': set()}
done = set()
done_link = set()
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
return nodes, links, meta
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
meta['objs'].add(obj_global_id)
nodes.add(obj_global_id)
for rel in get_obj_relationships(obj_global_id):
for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
meta['objs'].add(rel['id'])
if rel['id'] in done:
@ -99,13 +161,119 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
if level > 0:
next_level = level - 1
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
# done.add(rel['id'])
# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
if __name__ == '__main__':
source = ''
target = ''
add_obj_relationship(source, target, 'forward')
# print(get_obj_relationships(source))
def get_chat_forward_stats_out(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
chat_mess = rel['source'].split('/')
chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
if chat_target not in nb:
nb[chat_target] = 0
nb[chat_target] += 1
return nb
def get_chat_forward_stats_in(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
if r:
if not r[0]['target'] in nb:
nb[r[0]['target']] = 0
nb[r[0]['target']] += 1
# chat_mess = rel['source'].split('/')
#
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
# print(chat_target, chat, chat_target == chat)
# if chat is None or chat_target == chat:
# if chat_target not in nb:
# nb[chat_target] = 0
# nb[chat_target] += 1
#
# print(json.dumps(nb, indent=4))
return nb
def get_chat_forward_stats(obj_global_id): # objs_hidden
data = []
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
in_mess = get_chat_forward_stats_out(obj_global_id)
out_mess = get_chat_forward_stats_in(obj_global_id)
# if rel['target'] == obj_global_id:
# # print(rel)
# nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
#
for target in in_mess:
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
for source in out_mess:
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
# print()
# print(in_mess)
# print()
# print(out_mess)
return data
##################################################################
def get_chat_mention_stats_in(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
chat_mess = rel['source'].split('/')
chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
if chat_source not in nb:
nb[chat_source] = 0
nb[chat_source] += 1
return nb
def get_chat_mention_stats_out(obj_global_id):
nb = {}
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
if r:
if not r[0]['target'] in nb:
nb[r[0]['target']] = 0
nb[r[0]['target']] += 1
# chat_mess = rel['source'].split('/')
#
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
# print(chat_target, chat, chat_target == chat)
# if chat is None or chat_target == chat:
# if chat_target not in nb:
# nb[chat_target] = 0
# nb[chat_target] += 1
#
# print(json.dumps(nb, indent=4))
return nb
def get_chat_mentions_stats(obj_global_id): # objs_hidden
data = []
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
out_mess = get_chat_mention_stats_in(obj_global_id)
in_mess = get_chat_mention_stats_out(obj_global_id)
#
for target in in_mess:
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
for source in out_mess:
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
# print()
# print(in_mess)
# print()
# print(out_mess)
return data

View file

@ -92,7 +92,13 @@ CURRENCIES = {
'regex': r'\b(?<![+/=])X[A-Za-z0-9]{33}(?![+/=])\b',
'max_execution_time': default_max_execution_time,
'tag': 'infoleak:automatic-detection="dash-address"',
}
},
'tron': {
'name': 'tron', # e.g. TYdds9VLDjUshf9tbsXSfGUZNzJSbbBeat
'regex': r'\b(?<![+/=])T[0-9a-zA-Z]{33}(?![+/=])\b',
'max_execution_time': default_max_execution_time,
'tag': 'infoleak:automatic-detection="tron-address"',
},
}
##################################
##################################

View file

@ -263,3 +263,9 @@ def sanitise_daterange(date_from, date_to, separator='', date_type='str'):
date_from = date_to
date_to = res
return date_from, date_to
def get_previous_month_date():
now = datetime.date.today()
first = now.replace(day=1)
last_month = first - datetime.timedelta(days=1)
return last_month.strftime("%Y%m%d")

View file

@ -297,6 +297,44 @@ def objects_user_account():
ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
translation_languages=languages, translation_target=target)
@chats_explorer.route("/objects/user-account_chats_chord_json", methods=['GET']) # TODO API
@login_required
@login_read_only
def objects_user_account_chats_chord_json():
subtype = request.args.get('subtype')
user_id = request.args.get('id')
json_graph = chats_viewer.get_user_account_chats_chord(subtype, user_id)
return jsonify(json_graph)
@chats_explorer.route("/objects/user-account_mentions_chord_json", methods=['GET']) # TODO API
@login_required
@login_read_only
def objects_user_account_mentions_chord_json():
subtype = request.args.get('subtype')
user_id = request.args.get('id')
json_graph = chats_viewer.get_user_account_mentions_chord(subtype, user_id)
return jsonify(json_graph)
@chats_explorer.route("/objects/user-account/chat", methods=['GET'])
@login_required
@login_read_only
def objects_user_account_chat():
instance_uuid = request.args.get('subtype')
user_id = request.args.get('id')
chat_id = request.args.get('chat_id')
target = request.args.get('target')
if target == "Don't Translate":
target = None
meta = chats_viewer.api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=target)
if meta[1] != 200:
return create_json_response(meta[0], meta[1])
else:
meta = meta[0]
languages = Language.get_translation_languages()
return render_template('chats_explorer/user_chat_messages.html', meta=meta, bootstrap_label=bootstrap_label,
ail_tags=Tag.get_modal_add_tags(meta['user-account']['id'], meta['user-account']['type'], meta['user-account']['subtype']),
translation_languages=languages, translation_target=target)
@chats_explorer.route("objects/user-account/messages/stats/week/all", methods=['GET'])
@login_required
@login_read_only

View file

@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.objects import ail_objects
from lib import chats_viewer
from lib import Tag
bootstrap_label = Flask_config.bootstrap_label
@ -263,9 +264,37 @@ def relationships_graph_node_json():
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
return jsonify(json_graph)
@correlation.route('/relationships/chord_graph_json')
@login_required
@login_read_only
def relationships_chord_graph_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationships/chord_mentions_graph_json')
@login_required
@login_read_only
def relationships_chord_mentions_graph_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
return jsonify({'meta': meta, 'data': chat_json_graph})
@correlation.route('/relationship/show', methods=['GET', 'POST'])
@login_required
@ -278,8 +307,28 @@ def show_relationship():
max_nodes = request.form.get('max_nb_nodes_in')
level = sanitise_level(request.form.get('level'))
## get all selected relationships
relationships = []
for relationship in ail_objects.get_relationships():
rel_option = request.form.get(f'relationship_{relationship}_Check')
if rel_option:
relationships.append(relationship)
relationships = ",".join(relationships)
## get all selected objects types
filter_types = []
for ob_type in ail_objects.get_all_objects():
correl_option = request.form.get(f'{ob_type}_Check')
if correl_option:
filter_types.append(ob_type)
# list as params
filter_types = ",".join(filter_types)
# redirect to keep history and bookmark
return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
filter=filter_types, relationships=relationships,
max_nodes=max_nodes, level=level))
# request.method == 'GET'
@ -290,6 +339,9 @@ def show_relationship():
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
# check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
@ -300,6 +352,8 @@ def show_relationship():
"object_type": obj_type,
"max_nodes": max_nodes, "level": level,
"correlation_id": obj_id,
"relationships": relationships, "relationships_str": ",".join(relationships),
"filter": filter_types, "filter_str": ",".join(filter_types),
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
"nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
}

View file

@ -316,6 +316,17 @@ def crawlers_last_domains_month_json():
stats = crawlers.get_crawlers_stats_by_month(domain_type)
return jsonify(stats)
@crawler_splash.route('/crawlers/last/domains/month/previous/json')
@login_required
@login_read_only
def crawlers_last_domains_previous_month_json():
domain_type = request.args.get('type')
if domain_type not in crawlers.get_crawler_all_types():
return jsonify({'error': 'Invalid domain type'}), 400
date = Date.get_previous_month_date()
stats = crawlers.get_crawlers_stats_by_month(domain_type, date=date)
return jsonify(stats)
@crawler_splash.route('/crawlers/last/domains/status/month/json')
@login_required
@login_read_only
@ -800,7 +811,7 @@ def crawler_cookiejar_cookie_edit_post():
@login_required
@login_read_only
def crawler_cookiejar_cookie_add():
cookiejar_uuid = request.args.get('cookiejar_uuid')
cookiejar_uuid = request.args.get('uuid')
return render_template("add_cookie.html", cookiejar_uuid=cookiejar_uuid)
@ -831,7 +842,7 @@ def crawler_cookiejar_cookie_manual_add_post():
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
return redirect(url_for('crawler_splash.crawler_cookiejar_show', uuid=cookiejar_uuid))
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['POST'])

View file

@ -145,7 +145,7 @@ def tracked_menu_admin():
def show_tracker():
user_id = current_user.get_user_id()
tracker_uuid = request.args.get('uuid', None)
res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
res = Tracker.api_is_allowed_to_access_tracker(tracker_uuid, user_id)
if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@ -159,7 +159,7 @@ def show_tracker():
tracker = Tracker.Tracker(tracker_uuid)
meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
'user', 'webhook', 'nb_objs'})
'user', 'webhooks', 'nb_objs'})
if meta['type'] == 'yara':
yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
@ -300,6 +300,7 @@ def add_tracked_menu():
return create_json_response(res[0], res[1])
else:
return render_template("tracker_add.html",
dict_tracker={},
all_sources=item_basic.get_all_items_sources(r_list=True),
tags_selector_data=Tag.get_tags_selector_data(),
all_yara_files=Tracker.get_all_default_yara_files())
@ -314,6 +315,8 @@ def tracker_edit():
res = Tracker.api_edit_tracker(input_dict, user_id)
if res[1] == 200:
return redirect(url_for('hunters.show_tracker', uuid=res[0].get('uuid')))
else:
return create_json_response(res[0], res[1])
else:
user_id = current_user.get_user_id()
tracker_uuid = request.args.get('uuid', None)
@ -322,10 +325,16 @@ def tracker_edit():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
tracker = Tracker.Tracker(tracker_uuid)
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhook'})
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'tags', 'webhooks'})
if dict_tracker['type'] == 'yara':
if not Tracker.is_default_yara_rule(dict_tracker['tracked']):
dict_tracker['content'] = Tracker.get_yara_rule_content(dict_tracker['tracked'])
elif dict_tracker['type'] == 'set':
tracked, nb_words = dict_tracker['tracked'].rsplit(';', 1)
tracked = tracked.replace(',', ' ')
dict_tracker['tracked'] = tracked
dict_tracker['nb_words'] = nb_words
taxonomies_tags, galaxies_tags, custom_tags = Tag.sort_tags_taxonomies_galaxies_customs(dict_tracker['tags'])
tags_selector_data = Tag.get_tags_selector_data()
tags_selector_data['taxonomies_tags'] = taxonomies_tags

View file

@ -0,0 +1,140 @@
//Requirement: - D3v7
// - jquery
// container_id = #container_id{"meta": {obj_gid: label, ...}, "data": [{"source": source, "target": target, "value": value}, ...]}
// tooltip = d3 tooltip object
// TODO: - Mouseover object
const create_directed_chord_diagram = (container_id, data, min_value= 0, max_value = -1, fct_mouseover, fct_mouseout, options) => {
// Filter data by value between target and source
if (min_value > 0) {
data.data = data.data.filter(function(value) {
return data.value >= min_value;
});
}
if (max_value > 0) {
data.data = data.data.filter(function(value) {
return data.value <= max_value;
});
}
function getMaxCharsToShow(angle, radius) {
const approximateCharWidth = 7; // Approximate width of a character in pixels
const arcLength = angle * radius; // Length of the arc
return Math.floor(arcLength / approximateCharWidth); // Maximum number of characters that can fit
}
const width = 900;
const height = width;
const innerRadius = Math.min(width, height) * 0.5 - 20;
const outerRadius = innerRadius + 6;
const labels_meta = data.meta
data = data.data
// Compute a dense matrix from the weighted links in data.
var names = Array.from(d3.union(data.flatMap(d => [d.source, d.target])));
const index = new Map(names.map((name, i) => [name, i]));
const matrix = Array.from(index, () => new Array(names.length).fill(0));
for (const {source, target, value} of data) matrix[index.get(source)][index.get(target)] += value;
const chord = d3.chordDirected()
.padAngle(12 / innerRadius)
.sortSubgroups(d3.descending)
.sortChords(d3.descending);
const arc = d3.arc()
.innerRadius(innerRadius)
.outerRadius(outerRadius);
const ribbon = d3.ribbonArrow()
.radius(innerRadius - 0.5)
.padAngle(1 / innerRadius);
const colors = d3.schemeCategory10;
const svg = d3.select(container_id)
.append("svg")
.attr("width", width)
.attr("height", height)
.attr("viewBox", [-width / 2, -height / 2, width, height])
.attr("style", "width: 100%; height: auto; font: 10px sans-serif;");
const chords = chord(matrix);
const textId = `text-${Math.random().toString(36).substring(2, 15)}`;
svg.append("path")
.attr("id", textId)
.attr("fill", "none")
.attr("d", d3.arc()({outerRadius, startAngle: 0, endAngle: 2 * Math.PI}));
svg.append("g")
.attr("fill-opacity", 0.75)
.selectAll("path")
.data(chords)
.enter()
.append("path")
.attr("d", ribbon)
.attr("fill", d => colors[d.target.index])
.style("mix-blend-mode", "multiply")
.append("title")
.text(d => `${labels_meta[names[d.source.index]]}
=> ${d.source.value} Messages
${labels_meta[names[d.target.index]]}`);
const g = svg.append("g")
.selectAll("g")
.data(chords.groups)
.enter()
.append("g");
g.append("path")
.attr("d", arc)
.attr("fill", d => colors[d.index])
.attr("stroke", "#fff")
.on("mouseover", function(event, d) {
fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]]);
})
.on("mouseout", fct_mouseout);
g.each(function(d) {
const group = d3.select(this);
const angle = d.endAngle - d.startAngle;
const text = labels_meta[names[d.index]];
const maxCharsToShow = getMaxCharsToShow(angle, outerRadius);
let displayedText
if (maxCharsToShow <= 1) {
displayedText = text[0];
} else {
displayedText = text.length > maxCharsToShow ? text.slice(0, maxCharsToShow - 1) + "…" : text;
}
let info_text = "OUT: " + d3.sum(matrix[d.index]) + " ; IN: " + d3.sum(matrix, row => row[d.index]) + " Messages"
if (displayedText) {
group.append("text")
.attr("dy", -3)
.append("textPath")
.attr("xlink:href", `#${textId}`)
.attr("startOffset", d.startAngle * outerRadius)
.on("mouseover", function(event, d) {
fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]], info_text);
})
.on("mouseout", fct_mouseout)
.text(displayedText);
}
});
//return svg.node();
}
// return svg ???
// return svg

View file

@ -1,4 +1,4 @@
//Requirement: - D3v5
//Requirement: - D3v7
// - jquery
// data input: var data = [{"count":0,"date":"2023-11-20","day":0,"hour":0}
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
@ -33,10 +33,12 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// create a tooltip
const tooltip = d3.select(container_id)
const tooltip = d3.select("body")
.append("div")
.attr("class", "tooltip_heatmap")
.style("opacity", 0)
.attr("class", "tooltip")
.style("position", "absolute")
.style("pointer-events", "none")
.style("background-color", "white")
.style("border", "solid")
.style("border-width", "2px")
@ -44,24 +46,28 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.style("padding", "5px")
// Three function that change the tooltip when user hover / move / leave a cell
const mouseover = function(d) {
tooltip.style("opacity", 1)
d3.select(this)
const mouseover = function(event, d) {
d3.select(event.target)
.style("stroke", "black")
//.style("stroke-opacity", 1)
.style("stroke-opacity", 1)
var xPosition = d3.mouse(this)[0] + margin.left;
var yPosition = d3.mouse(this)[1] + margin.top + window.scrollY + 100;
let d3_pageX = event.pageX;
let d3_pageY = event.pageY;
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
.style("left", xPosition + "px")
.style("top", yPosition + "px");
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
tooltip.transition()
.duration(200)
.style("opacity", 1);
}
const mouseleave = function(d) {
tooltip.style("opacity", 0)
d3.select(this)
const mouseleave = function(event, d) {
d3.select(event.target)
.style("stroke", "white")
//.style("stroke-opacity", 0.8)
tooltip.transition()
.duration(200)
.style("opacity", 0)
}
///////////////////////////////////////////////////////////////////////////
@ -114,22 +120,22 @@ const create_heatmap_week_hour = (container_id, data, options) => {
var heatMap = svg.selectAll(".hour")
.data(data)
.enter().append("rect")
.attr("x", function (d) {
return (d.hour - 1) * gridSize;
})
.attr("y", function (d) {
return (d.day - 1) * gridSize;
})
.attr("class", "hour bordered")
.attr("width", gridSize)
.attr("height", gridSize)
.style("stroke", "white")
.style("stroke-opacity", 0.6)
.style("fill", function (d) {
return colorScale(d.count);
})
.on("mouseover", mouseover)
.on("mouseleave", mouseleave);
.attr("x", function (d) {
return (d.hour - 1) * gridSize;
})
.attr("y", function (d) {
return (d.day - 1) * gridSize;
})
.attr("class", "hour bordered")
.attr("width", gridSize)
.attr("height", gridSize)
.style("stroke", "white")
.style("stroke-opacity", 0.6)
.style("fill", function (d) {
return colorScale(d.count);
})
.on("mouseover", mouseover)
.on("mouseleave", mouseleave);
//Append title to the top
svg.append("text")
@ -222,7 +228,6 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.attr("transform", "translate(0," + (10) + ")")
.call(xAxis);
// return svg ???
// return svg

View file

@ -1,4 +1,4 @@
// sanitise str_to_sanitize
function sanitize_text(str_to_sanitize) {
return $("<span\>").text(str_to_sanitize).html()
};
}

View file

@ -85,7 +85,7 @@
<i class="fas fa-user-circle"></i>
<i class="far fa-comment-dots"></i>
</span>
{{ meta["nb_messages"] }}&nbsp;&nbsp;
<a class="badge-primary px-1 py-0" href="{{ url_for('chats_explorer.objects_user_account_chat') }}?subtype={{ meta['subtype'] }}&id={{ main_obj_id }}&chat_id={{ meta['id'] }}">{{ meta["nb_messages"] }}&nbsp;&nbsp;</a>
</span>
{% endif %}
</div>

View file

@ -95,6 +95,10 @@
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>
<a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships &nbsp;
</button>
</a>
{% endif %}
</span>

View file

@ -87,6 +87,11 @@
{% else %}
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>
<a href="{{ url_for('correlation.show_relationship')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-secondary"><i class="far fa-eye"></i> Relationships &nbsp;
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>

View file

@ -12,12 +12,14 @@
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
</head>
@ -34,6 +36,9 @@
{% include 'chats_explorer/card_user_account.html' %}
<span class="mt-3">
{% include 'objects/image/block_blur_img_slider.html' %}
</span>
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
@ -42,10 +47,16 @@
<h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>
<div id="heatmapweekhourall"></div>
<h4>Numbers of Messages Posted by Chat:</h4>
<div id="chord_user_chats" style="max-width: 900px"></div>
<h4>Numbers of Mentions:</h4>
<div id="chord_mentions" style="max-width: 900px"></div>
<h4>User Chats:</h4>
{% for meta_chats in meta['chats'] %}
<div class="my-2">
{% with meta=meta_chats %}
{% with meta=meta_chats,main_obj_id=meta['id'] %}
{% include 'chats_explorer/basic_card_chat.html' %}
{% endwith %}
</div>
@ -58,6 +69,8 @@
</div>
</div>
{% include 'objects/tooltip_ail_objects.html' %}
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
@ -77,26 +90,19 @@ d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?su
create_heatmap_week_hour('#heatmapweekhourall', data);
})
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
let url = "{{ url_for('chats_explorer.objects_user_account_chats_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
d3.json(url).then(function(data) {
create_directed_chord_diagram('#chord_user_chats', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
let url2 = "{{ url_for('chats_explorer.objects_user_account_mentions_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
d3.json(url2).then(function(data) {
create_directed_chord_diagram('#chord_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
</script>
</body>
</html>

View file

@ -0,0 +1,133 @@
<!DOCTYPE html>
<html>
<head>
<title>User Account - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<h3>User:</h3>
{% with meta=meta['user-account'] %}
{% include 'chats_explorer/card_user_account.html' %}
{% endwith %}
<h3>Chat:</h3>
{% with meta=meta['chat'] %}
{% include 'chats_explorer/basic_card_chat.html' %}
{% endwith %}
<div class="mt-2">
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% include 'objects/image/block_blur_img_slider.html' %}
</div>
<div class="position-relative">
<div class="chat-messages p-2">
{% for date in meta['messages'] %}
<div class="divider d-flex align-items-center mb-4">
<p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
<span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
</p>
</div>
{% for mess in meta['messages'][date] %}
{% with message=mess %}
{% include 'chats_explorer/block_message.html' %}
{% endwith %}
{% endfor %}
<br>
{% endfor %}
</div>
</div>
{# {% if meta['chats'] %}#}
{# <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>#}
{# <div id="heatmapweekhourall"></div>#}
{##}
{# <h4>User Chats:</h4>#}
{# {% for meta_chats in meta['chats'] %}#}
{# <div class="my-2">#}
{# {% with meta=meta_chats %}#}
{# {% include 'chats_explorer/basic_card_chat.html' %}#}
{# {% endwith %}#}
{# </div>#}
{# {% endfor %}#}
{# {% endif %}#}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
});
{#d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}")#}
{# .then(function(data) {#}
{# create_heatmap_week_hour('#heatmapweekhourall', data);#}
{# })#}
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

View file

@ -16,7 +16,8 @@
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.v7.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
<style>
.icon_legend {
@ -71,6 +72,7 @@
.blured {
filter: blur(5px);
max-width: 200px;
}
.graph_panel {
@ -127,6 +129,10 @@
{% include 'correlation/metadata_card_item.html' %}
{% endif %}
<div class="my-2">
{% include 'objects/image/block_blur_img_slider.html' %}
</div>
<div class="row">
<div class="col-xl-10">
@ -205,13 +211,49 @@
<div class="card-body text-center px-0 py-0">
<ul class="list-group">
<li class="list-group-item list-group-item-info">Relationship</li>
<li class="list-group-item list-group-item-info">Relationships</li>
<form action="{{ url_for('correlation.show_relationship') }}" method="post">
<li class="list-group-item text-left">
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_from_Check" name="relationship_forwarded_from_Check" {%if "forwarded_from" in dict_object["relationships"]%}checked{%endif%}>
<label class="form-check-label" for="relationship_forwarded_from_Check">Forwarded From</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_to_Check" name="relationship_forwarded_to_Check" {%if "forwarded_to" in dict_object["relationships"]%}checked{%endif%}>
<label class="form-check-label" for="relationship_forwarded_to_Check">Forwarded To</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="relationship_in_Check" name="relationship_in_Check" {%if "in" in dict_object["relationships"]%}checked{%endif%}>
<label class="form-check-label" for="relationship_in_Check">In</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
<label class="form-check-label" for="relationship_mention_Check">Mention</label>
</div>
</li>
<input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
<input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
<li class="list-group-item list-group-item-info">Objects Types</li>
<li class="list-group-item text-left">
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="chat_Check">Chat</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="message_Check">Message</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="user-account_Check">User-Account</label>
</div>
</li>
{# <li class="list-group-item text-left">#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
@ -313,10 +355,25 @@
</div>
</div>
<h3>Forwards</h3>
<div id="chart_test" style="max-width: 900px"></div>
<h3>Mentions</h3>
<div id="chart_mentions" style="max-width: 900px"></div>
<table id="table_graph_node_objects">
</table>
<div id="timeline"></div>
</div>
</div>
</div>
{% include 'objects/tooltip_ail_objects.html' %}
<script>
var all_graph = {};
@ -324,8 +381,20 @@ $(document).ready(function(){
$("#incomplete_graph").hide();
$("#page-Decoded").addClass("active");
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
all_graph.onResize();
let url = "{{ url_for('correlation.relationships_chord_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
d3.json(url).then(function(data) {
create_directed_chord_diagram('#chart_test', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
let url2 = "{{ url_for('correlation.relationships_chord_mentions_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
d3.json(url2).then(function(data) {
create_directed_chord_diagram('#chart_mentions', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
});
});
const blur_slider_correlation = $('#blur-slider-correlation');
@ -553,30 +622,41 @@ d3.json(url)
d3.select("body").on("keypress", keypressed)
let table_obj = document.getElementById("table_graph_node_objects")
for (let i=0; i<data.nodes.length; i++) {
let newRow = table_obj.insertRow();
let newCell = newRow.insertCell();
let newText = document.createTextNode(data.nodes[i].id);
newCell.appendChild(newText);
//console.log(data.nodes[i])
}
//// --------------------------------------------------------------------------------------------------------
})
.catch(function(error) {
$("#graph_loading").remove()
svg_node.remove();
d3.select("#graph").append("div")
.text(error);
.text(error);
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
function zoomed(event, d) {
container_graph.attr("transform", event.transform);
}
function doubleclick (d) {
function doubleclick (event, d) {
window.open(d.url, '_blank');
}
function keypressed () {
//console.log(d3.event.keyCode)
function keypressed (event, d) {
//console.log(event.keyCode)
//console.log(currentObject.id)
// hide node, H or h key
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
if ((event.keyCode === 72 || event.keyCode === 104) && currentObject) {
window.location.href = correl_link + "&hide=" + currentObject.id
}
@ -586,29 +666,29 @@ function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
function drag_start(event, d) {
if (!event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
function dragged(event, d) {
d.fx = event.x;
d.fy = event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
function drag_end(event, d) {
if (!event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
function mouseovered(event, d, obj_gid, obj_label, additional_text) {
currentObject = d;
var d3_pageX = d3.event.pageX;
var d3_pageY = d3.event.pageY;
var d3_pageX = event.pageX;
var d3_pageY = event.pageY;
if (d.popover) {
div.html(d.popover)
@ -621,8 +701,16 @@ if (d.popover) {
blur_tooltip();
} else {
var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
if (obj_label) {
pop_header = pop_header + sanitize_text(obj_label)
} else if (obj_gid) {
pop_header = pop_header + sanitize_text(obj_gid)
} else {
pop_header = pop_header + sanitize_text(d.text)
}
pop_header = pop_header + "</h6></div>"
let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
div.html(pop_header + spinner)
.style("left", (d3_pageX) + "px")
@ -632,9 +720,16 @@ if (d.popover) {
.duration(200)
.style("opacity", 1);
$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
let description_url = "{{ url_for('correlation.get_description') }}?object_id="
if (obj_gid) {
description_url = description_url + obj_gid
} else {
description_url = description_url + d.id
}
$.getJSON(description_url,
function(data){
var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
let desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
Object.keys(data).forEach(function(key) {
if (key=="status") {
desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
@ -650,9 +745,16 @@ if (d.popover) {
desc = desc + "fa-times-circle\"></i>DOWN"
}
desc = desc + "</div></dd>"
} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
}
} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
if (data[key]) {
if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
let date = sanitize_text(data[key])
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
} else {
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
}
}
}
});
desc = desc + "</dl>"
@ -674,6 +776,9 @@ if (d.popover) {
desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
}
}
if (additional_text) {
desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
}
desc = desc + "</div></div>"
div.html(desc)
@ -697,7 +802,6 @@ if (d.popover) {
function mouseouted() {
currentObject = null;
div.transition()
.duration(500)
.style("opacity", 0);
@ -712,8 +816,18 @@ all_graph.onResize = function () {
}
window.all_graph = all_graph;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*
*/
</script>
</body>
</body>
</html>

View file

@ -95,6 +95,10 @@
<div id="barchart_type_month"></div>
<div class="text-center" id="pie_chart_month"></div>
<h3>Previous Month Stats:</h3>
<div id="barchart_type_month"></div>
<div class="text-center" id="barchart_type_previous_month"></div>
</div>
</div>
@ -156,6 +160,13 @@ $(document).ready(function(){
}
);
$.getJSON("{{ url_for('crawler_splash.crawlers_last_domains_previous_month_json') }}?type={{type}}",
function (data) {
let div_width = $("#barchart_type_previous_month").width();
barchart_stack("barchart_type_previous_month", data, {"width": div_width});
}
);
});
function toggle_sidebar(){

View file

@ -75,7 +75,7 @@
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" {% if not dict_tracker['filters'] or 'decoded' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Content that has been decoded from an encoded format, such as base64"></i></label>
</div>
{# <div class="custom-control custom-switch mt-1">#}
@ -83,7 +83,7 @@
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" {% if not dict_tracker['filters'] or 'item' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Text that has been processed by AIL. It can include various types of extracted information"></i></label>
</div>
<div class="card border-dark mb-4" id="sources_item_div">
@ -100,14 +100,14 @@
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" {% if not dict_tracker['filters'] or 'pgp' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="PGP key/block metadata"></i></label>
</div>
<div class="card border-dark mb-4" id="sources_pgp_div">
<div class="card-body">
<h6>Filter PGP by subtype:</h6>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'name' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
<label class="custom-control-label" for="filter_pgp_name">
<svg height="26" width="26">
<g class="nodes">
@ -119,7 +119,7 @@
</label>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" {% if not dict_tracker['filters'] %}checked=""{% endif %} {% if 'pgp' in dict_tracker['filters'] %}{% if not 'subtypes' in dict_tracker['filters']['pgp'] %}checked=""{% else %}{% if 'mail' in dict_tracker['filters']['pgp']['subtypes'] %}checked=""{% endif %}{% endif %}{% endif %}>
<label class="custom-control-label" for="filter_pgp_mail">
<svg height="26" width="26">
<g class="nodes">
@ -133,15 +133,15 @@
</div>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" checked="">
<input class="custom-control-input" type="checkbox" name="title_obj" id="title_obj" {% if not dict_tracker['filters'] or 'title' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="title_obj"><i class="fas fa-heading"></i>&nbsp;Title <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Title that has been extracted from a HTML page"></i></label>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
<input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" {% if not dict_tracker['filters'] or 'message' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i>&nbsp;Message <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Messages from Chats"></i></label>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" checked="">
<input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" {% if not dict_tracker['filters'] or 'ocr' in dict_tracker['filters'] %}checked=""{% endif %}>
<label class="custom-control-label" for="ocr_obj"><i class="fas fa-comment-dots"></i>&nbsp;OCR <i class="fas fa-expand text-info" data-toggle="tooltip" data-placement="right" title="Text extracted from Images"></i></label>
</div>
@ -310,6 +310,9 @@ $(document).ready(function(){
emptyText: 'Item Sources to track (ALL IF EMPTY)',
});
item_source_input_controller();
pgp_source_input_controller();
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {

View file

@ -0,0 +1,136 @@
<script>
let tooltip_ail_obj = d3.select("body").append("div")
//.attr("class", "tooltip_graph")
.attr("id", "tooltip_graph")
.style("opacity", 0)
.style("position", "absolute")
.style("text-align", "center")
.style("padding", "2px")
.style("font", "12px sans-serif")
.style("background", "#ebf4fb")
.style("border", "2px solid #b7ddf2")
.style("border-radius", "8px")
.style("pointer-events", "none")
.style("color", "#000000");
function mouseover_tooltip_ail_obj(event, d, obj_gid, obj_label, additional_text) { /// div var/const tooltip tooltip_ail_obj
let d3_pageX = event.pageX;
let d3_pageY = event.pageY;
if (d.popover) {
tooltip_ail_obj.html(d.popover)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
tooltip_ail_obj.transition()
.duration(200)
.style("opacity", 1);
blur_images();
} else {
let pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
if (obj_label) {
pop_header = pop_header + sanitize_text(obj_label)
} else if (obj_gid) {
pop_header = pop_header + sanitize_text(obj_gid)
} else {
pop_header = pop_header + sanitize_text(d.text)
}
pop_header = pop_header + "</h6></div>"
let spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
tooltip_ail_obj.html(pop_header + spinner)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
tooltip_ail_obj.transition()
.duration(200)
.style("opacity", 1);
let description_url = "{{ url_for('correlation.get_description') }}?object_id="
if (obj_gid) {
description_url = description_url + obj_gid
} else {
description_url = description_url + d.id
}
let desc
$.getJSON(description_url, function(data){
desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
Object.keys(data).forEach(function(key) {
if (key==="status") {
desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
if (data["status"]) {
desc = desc + "Green"
} else {
desc = desc + "Red"
}
desc = desc + ";\"><i class=\"fas "
if (data["status"]) {
desc = desc + "fa-check-circle\"></i>UP"
} else {
desc = desc + "fa-times-circle\"></i>DOWN"
}
desc = desc + "</div></dd>"
} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
if (data[key]) {
if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
let date = sanitize_text(data[key])
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
} else {
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
}
}
}
});
desc = desc + "</dl>"
if (data["tags"]) {
data["tags"].forEach(function(tag) {
desc = desc + "<span class=\"badge badge-warning\">"+ sanitize_text(tag) +"</span>";
});
}
if (data["img"]) {
if (data["tags_safe"]) {
if (data["type"] === "screenshot") {
desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
} else {
desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
}
desc = desc + data["img"] +" class=\"object_image\" id=\"tooltip_screenshot_correlation\" style=\"filter: blur(5px);max-width: 200px;\"/>";
} else {
desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
}
}
if (additional_text) {
desc = desc + "<hr><div>" + sanitize_text(additional_text) + "</div>"
}
desc = desc + "</div></div>"
tooltip_ail_obj.html(desc)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
d.popover = desc
if (data["img"]) {
blur_images();
}
})
.fail(function(error) {
desc = pop_header + "<div class=\"card-body bg-dark pt-0\"><i class=\"fas fa-3x fa-times text-danger\"></i>"+ error.statusText +"</div>"
tooltip_ail_obj.html(desc)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
});
}
}
function mouseout_tooltip_ail_obj(event, d) {
tooltip_ail_obj.transition()
.duration(500)
.style("opacity", 0);
}
</script>

View file

@ -9,7 +9,9 @@ wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.
BOOTSTRAP_VERSION='4.2.1'
FONT_AWESOME_VERSION='5.7.1'
D3_JS_VERSION='5.16.0'
wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
rm -rf temp
mkdir temp