diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py
index 70a96fa5..f342cd01 100755
--- a/bin/importer/feeders/Default.py
+++ b/bin/importer/feeders/Default.py
@@ -44,6 +44,11 @@ class DefaultFeeder:
def get_date(self):
return datetime.date.today().strftime("%Y%m%d")
+ def get_feeder_timestamp(self):
+ timestamp = self.json_data.get('timestamp')
+ if timestamp:
+ return int(timestamp)
+
def get_json_data(self):
"""
Return the JSON data,
diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py
index b48fe4ad..3b71a72e 100755
--- a/bin/importer/feeders/abstract_chats_feeder.py
+++ b/bin/importer/feeders/abstract_chats_feeder.py
@@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
import datetime
import os
import sys
+import time
from abc import ABC
@@ -127,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta'].get('reply_to', {}).get('message_id')
def get_message_forward(self):
- return self.json_data['meta'].get('forward')
+ return self.json_data['meta'].get('forward', {})
def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data'])
@@ -164,7 +165,40 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
self.obj = Messages.Message(obj_id)
return self.obj
- def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
+ # TODO handle subchannel
+ def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
+ chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
+
+ # Obj Daterange
+ chat.add(date)
+
+ if meta_chat.get('name'):
+ chat.set_name(meta_chat['name'])
+
+ if meta_chat.get('info'):
+ chat.set_info(meta_chat['info'])
+
+ if meta_chat.get('date'): # TODO check if already exists
+ chat.set_created_at(int(meta_chat['date']['timestamp']))
+
+ if meta_chat.get('icon'):
+ img = Images.create(meta_chat['icon'], b64=True)
+ img.add(date, chat)
+ chat.set_icon(img.get_global_id())
+ if new_objs:
+ new_objs.add(img)
+
+ if meta_chat.get('username'):
+ username = Username(meta_chat['username'], self.get_chat_protocol())
+ chat.update_username_timeline(username.get_global_id(), int(time.time()))
+ username.add(date, obj=chat) # TODO TODAY DATE
+
+ return chat
+
+ ##############################################################################################################################
+
+
+ def process_chat(self, new_objs, obj, date, timestamp, feeder_timestamp, reply_id=None):
meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
subchannel = None
@@ -190,7 +224,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
- chat.update_username_timeline(username.get_global_id(), timestamp)
+ chat.update_username_timeline(username.get_global_id(), feeder_timestamp)
if meta.get('subchannel'):
subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
@@ -257,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# else:
# # ADD NEW MESSAGE REF (used by discord)
+ ##########################################################################################
+
+ def _process_user(self, meta, date, timestamp, new_objs=None):
+ user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
+ if meta.get('username'):
+ username = Username(meta['username'], self.get_chat_protocol())
+ # TODO timeline or/and correlation ????
+ user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+ user_account.update_username_timeline(username.get_global_id(), timestamp)
+
+ # Username---Message
+ username.add(date) # TODO # correlation message ??? ###############################################################
+
+ # ADDITIONAL METAS
+ if meta.get('firstname'):
+ user_account.set_first_name(meta['firstname'])
+ if meta.get('lastname'):
+ user_account.set_last_name(meta['lastname'])
+ if meta.get('phone'):
+ user_account.set_phone(meta['phone'])
+
+ if meta.get('icon'):
+ img = Images.create(meta['icon'], b64=True)
+ img.add(date, user_account)
+ user_account.set_icon(img.get_global_id())
+ new_objs.add(img)
+
+ if meta.get('info'):
+ user_account.set_info(meta['info'])
+
+ user_account.add(date)
+ return user_account
+
def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta'].get('sender')
if not meta:
@@ -305,8 +372,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if self.obj:
objs.add(self.obj)
new_objs = set()
+ chats_objs = set()
date, timestamp = self.get_message_date_timestamp()
+ feeder_timestamp = self.get_feeder_timestamp()
+ if not feeder_timestamp:
+ feeder_timestamp = timestamp
# REPLY
reply_id = self.get_message_reply_id()
@@ -363,81 +434,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
print(obj.id)
# CHAT
- chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
-
- # # TODO HANDLE OTHERS OBJECT TYPE
- # # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- # # Message forward + Discussion
- # if self.get_json_meta().get('forward'):
- # discussion_id = self.get_json_meta().get('discussion')
- # forward_from = self.get_message_forward()
- #
- # if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
- # chat_forward_id = forward_from['from']['id']
- # message_forward_id = forward_from['from']['channel_post']
- #
- # # if chat_forward_id == discussion_id:
- # # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
- # # if linked_chat.exists():
- # # # create thread
- # # # add message replies for each childrens
- #
- # # TODO HANDLE THREAD
- # # TODO Change FORWARD META FIELDS
- # # meta['forward'] = {}
- # # # CHAT ID
- # # # SUBCHANNEL ID -> can be None
- # # # Message ID
- #
- # # meta['forward']['origin']
- # # # same as 'forward'
- #
- # if self.get_json_meta().get('forward'):
- # forward = self.get_message_forward()
- # f_chat = forward['chat']
- # f_subchannel = forward.get('subchannel')
- # f_id = forward.get('id')
- # if not f_subchannel:
- # chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
- # if chat_forward.exists():
- # for chat_obj in chat_objs:
- # if chat_obj.type == 'chat':
- # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
- # # TODO LIST FORWARDED MESSAGES
- #
- #
- # # Discord -> serverID + subchannel ID + message ID
- # # Telegram -> chat ID + Message ID
- # # + ORIGIN IDs
- #
- #
- #
- # # TODO create relationships graph
- #
- #
- # # TODO REMOVE ME
- # # Message forward # TODO handle subchannel + message ID
- # if self.get_json_meta().get('forward'):
- # forward_from = self.get_message_forward()
- # print('-----------------------------------------------------------')
- # print(forward_from)
- # if forward_from:
- # forward_from_type = forward_from['from']['type']
- # if forward_from_type == 'channel' or forward_from_type == 'chat':
- # chat_forward_id = forward_from['from']['id']
- # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
- # if chat_forward.exists():
- # for chat_obj in chat_objs:
- # if chat_obj.type == 'chat':
- # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
- # # chat_forward.add_relationship(obj.get_global_id(), 'forward')
+ curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, feeder_timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER
- user_account = self.process_sender(new_objs, obj, date, timestamp)
+ user_account = self.process_sender(new_objs, obj, date, feeder_timestamp)
if user_account:
# UserAccount---ChatObjects
- for obj_chat in chat_objs:
+ for obj_chat in curr_chats_objs:
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
# if chat: # TODO Chat---Username correlation ???
@@ -449,4 +453,73 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# -> subchannel ?
# -> thread id ?
+ chats_objs.update(curr_chats_objs)
+
+ #######################################################################
+
+ ## FORWARD ##
+ chat_fwd = None
+ user_fwd = None
+ if self.get_json_meta().get('forward'):
+ meta_fwd = self.get_message_forward()
+ if meta_fwd.get('chat'):
+ chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+ if meta_fwd.get('user'):
+ user_fwd = self._process_user(meta_fwd['user'], date, feeder_timestamp, new_objs=new_objs) # TODO date, timestamp ???
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
+
+ # TODO chat_fwd -> message
+ if chat_fwd or user_fwd:
+ for obj in objs:
+ if obj.type == 'message':
+ if chat_fwd:
+ obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
+ if user_fwd:
+ obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ obj.add_relationship(chat_obj.get_global_id(), 'in')
+ # -FORWARD- #
+
+ ## MENTION ##
+ if self.get_json_meta().get('mentions'):
+ for mention in self.get_json_meta()['mentions'].get('chats', []):
+ m_obj = self._process_chat(mention, date, new_objs=new_objs)
+ if m_obj:
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+ # TODO PERF
+ # TODO Keep message obj + chat obj in global var
+ for obj in objs:
+ if obj.type == 'message':
+ obj.add_relationship(m_obj.get_global_id(), 'mention')
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+ for mention in self.get_json_meta()['mentions'].get('users', []):
+ m_obj = self._process_user(mention, date, feeder_timestamp, new_objs=new_objs) # TODO date ???
+ if m_obj:
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+ # TODO PERF
+ # TODO Keep message obj + chat obj in global var
+ for obj in objs:
+ if obj.type == 'message':
+ obj.add_relationship(m_obj.get_global_id(), 'mention')
+ for chat_obj in chats_objs:
+ if chat_obj.type == 'chat':
+ obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+ # -MENTION- #
+
return new_objs | objs
diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py
index a4e41c90..a0d7d10b 100755
--- a/bin/lib/Tracker.py
+++ b/bin/lib/Tracker.py
@@ -762,6 +762,9 @@ def delete_obj_trackers(obj_type, subtype, obj_id):
#### TRACKERS ACL ####
## LEVEL ##
+def is_tracker_global_level(tracker_uuid):
+ return int(r_tracker.hget(f'tracker:{tracker_uuid}', 'level')) == 1
+
def is_tracked_in_global_level(tracked, tracker_type):
for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
tracker = Tracker(tracker_uuid)
@@ -805,6 +808,19 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
return {"status": "error", "reason": "Access Denied"}, 403
return {"uuid": tracker_uuid}, 200
+
+def api_is_allowed_to_access_tracker(tracker_uuid, user_id):
+ if not is_valid_uuid_v4(tracker_uuid):
+ return {"status": "error", "reason": "Invalid uuid"}, 400
+ tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
+ if not tracker_creator:
+ return {"status": "error", "reason": "Unknown uuid"}, 404
+ user = User(user_id)
+ if not is_tracker_global_level(tracker_uuid):
+ if not user.is_in_role('admin') and user_id != tracker_creator:
+ return {"status": "error", "reason": "Access Denied"}, 403
+ return {"uuid": tracker_uuid}, 200
+
##-- ACL --##
#### FIX DB #### TODO ###################################################################
diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py
index 9483a3d0..5bcb6920 100755
--- a/bin/lib/ail_core.py
+++ b/bin/lib/ail_core.py
@@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
if obj_type == 'chat-thread':
return r_object.smembers(f'all_chat-thread:subtypes')
if obj_type == 'cryptocurrency':
- return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+ return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
if obj_type == 'pgp':
return ['key', 'mail', 'name']
if obj_type == 'username':
diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py
index 8e47ef8b..34ab6157 100755
--- a/bin/lib/chats_viewer.py
+++ b/bin/lib/chats_viewer.py
@@ -328,6 +328,31 @@ def get_username_meta_from_global_id(username_global_id):
username = Usernames.Username(username_id, instance_uuid)
return username.get_meta(options={'icon'})
+###############################################################################
+# TODO Pagination
+def list_messages_to_dict(l_messages_id, translation_target=None):
+ options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
+ meta = {}
+ curr_date = None
+ for mess_id in l_messages_id:
+ message = Messages.Message(mess_id[1:])
+ timestamp = message.get_timestamp()
+ date_day = message.get_date()
+ date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}'
+ if date_day != curr_date:
+ meta[date_day] = []
+ curr_date = date_day
+ meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
+ meta[date_day].append(meta_mess)
+
+ # if mess_dict.get('tags'):
+ # for tag in mess_dict['tags']:
+ # if tag not in tags:
+ # tags[tag] = 0
+ # tags[tag] += 1
+ # return messages, pagination, tags
+ return meta
+
# TODO Filter
## Instance type
## Chats IDS
@@ -354,7 +379,7 @@ def get_messages_iterator(filters={}):
# threads
for threads in chat.get_threads():
thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
- _, _ = thread._get_messages(nb=-1)
+ messages, _ = thread._get_messages(nb=-1)
for mess in messages:
message_id, _, message_id = mess[0].split(':', )
yield Messages.Message(message_id)
@@ -404,6 +429,15 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
meta.append(chat_meta)
return meta
+def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
+ meta = {}
+ chat = Chats.Chat(chat_id, subtype)
+ chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
+ if chat_meta['username']:
+ chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
+
+ meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None)
+ return meta
def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
week = {}
@@ -431,6 +465,60 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
nb_day += 1
return stats
+def get_user_account_chats_chord(subtype, user_id):
+ nb = {}
+ user_account = UsersAccount.UserAccount(user_id, subtype)
+ for chat_g_id in user_account.get_chats():
+ c_subtype, c_id = chat_g_id.split(':', 1)
+ chat = Chats.Chat(c_id, c_subtype)
+ nb[f'chat:{chat_g_id}'] = len(chat.get_user_messages(user_id))
+
+ user_account_gid = user_account.get_global_id() # # #
+ chord = {'meta': {}, 'data': []}
+ label = get_chat_user_account_label(user_account_gid)
+ if label:
+ chord['meta'][user_account_gid] = label
+ else:
+ chord['meta'][user_account_gid] = user_account_gid
+
+ for chat_g_id in nb:
+ label = get_chat_user_account_label(chat_g_id)
+ if label:
+ chord['meta'][chat_g_id] = label
+ else:
+ chord['meta'][chat_g_id] = chat_g_id
+ chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
+ return chord
+
+def get_user_account_mentions_chord(subtype, user_id):
+ chord = {'meta': {}, 'data': []}
+ nb = {}
+ user_account = UsersAccount.UserAccount(user_id, subtype)
+ user_account_gid = user_account.get_global_id()
+ label = get_chat_user_account_label(user_account_gid)
+ if label:
+ chord['meta'][user_account_gid] = label
+ else:
+ chord['meta'][user_account_gid] = user_account_gid
+
+ for mess in user_account.get_messages():
+ m = Messages.Message(mess[9:])
+ for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
+ if rel:
+ if not rel['target'] in nb:
+ nb[rel['target']] = 0
+ nb[rel['target']] += 1
+
+ for g_id in nb:
+ label = get_chat_user_account_label(g_id)
+ if label:
+ chord['meta'][g_id] = label
+ else:
+ chord['meta'][g_id] = g_id
+ chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
+ return chord
+
+
def _get_chat_card_meta_options():
return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
@@ -481,18 +569,65 @@ def fix_correlations_subchannel_message():
#### API ####
+def get_chat_user_account_label(chat_gid):
+ label = None
+ obj_type, subtype, obj_id = chat_gid.split(':', 2)
+ if obj_type == 'chat':
+ obj = get_obj_chat(obj_type, subtype, obj_id)
+ username = obj.get_username()
+ if username:
+ username = username.split(':', 2)[2]
+ name = obj.get_name()
+ if username and name:
+ label = f'{username} - {name}'
+ elif username:
+ label = username
+ elif name:
+ label = name
+
+ elif obj_type == 'user-account':
+ obj = UsersAccount.UserAccount(obj_id, subtype)
+ username = obj.get_username()
+ if username:
+ username = username.split(':', 2)[2]
+ name = obj.get_name()
+ if username and name:
+ label = f'{username} - {name}'
+ elif username:
+ label = username
+ elif name:
+ label = name
+ return label
+
+def enrich_chat_relationships_labels(relationships):
+ meta = {}
+ for row in relationships:
+ if row['source'] not in meta:
+ label = get_chat_user_account_label(row['source'])
+ if label:
+ meta[row['source']] = label
+ else:
+ meta[row['source']] = row['source']
+
+ if row['target'] not in meta:
+ label = get_chat_user_account_label(row['target'])
+ if label:
+ meta[row['target']] = label
+ else:
+ meta[row['target']] = row['target']
+ return meta
def api_get_chat_service_instance(chat_instance_uuid):
chat_instance = ChatServiceInstance(chat_instance_uuid)
if not chat_instance.exists():
return {"status": "error", "reason": "Unknown uuid"}, 404
return chat_instance.get_meta({'chats'}), 200
-def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
+def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
# print(chat.get_obj_language_stats())
- meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
+ meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target)
if meta['username']:
meta['username'] = get_username_meta_from_global_id(meta['username'])
if meta['subchannels']:
@@ -500,7 +635,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
else:
if translation_target not in Language.get_translation_languages():
translation_target = None
- meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
+ if messages:
+ meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id):
@@ -602,6 +738,18 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
return meta, 200
+def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None):
+ user_account = UsersAccount.UserAccount(user_id, instance_uuid)
+ if not user_account.exists():
+ return {"status": "error", "reason": "Unknown user-account"}, 404
+ meta = get_user_account_chat_message(user_id, instance_uuid, chat_id)
+ meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
+ resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False)
+ if resp[1] != 200:
+ return resp
+ meta['chat'] = resp[0]
+ return meta, 200
+
def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
if not user_account.exists():
diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py
index 3cecb779..1142163c 100755
--- a/bin/lib/crawlers.py
+++ b/bin/lib/crawlers.py
@@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
return schedule_uuid
+def _delete_schedules():
+ for schedule_uuid in get_schedulers_uuid():
+ schedule = CrawlerSchedule(schedule_uuid)
+ schedule.delete()
+
# TODO sanityze UUID
def api_delete_schedule(data):
schedule_uuid = data.get('uuid')
@@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
external=external)
return task_uuid
-
## -- CRAWLER TASK -- ##
#### CRAWLER TASK API ####
diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py
index 01ff0c5e..ac545cc2 100755
--- a/bin/lib/objects/CryptoCurrencies.py
+++ b/bin/lib/objects/CryptoCurrencies.py
@@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject):
pass
def is_valid_address(self):
- if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin':
+ if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron':
return check_base58_address(self.id)
else:
return True
@@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject):
return 'ZEC'
elif self.subtype == 'dash':
return 'DASH'
+ elif self.subtype == 'tron':
+ return 'TRX'
return None
def get_link(self, flask_context=False):
@@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject):
def get_all_subtypes():
# return ail_core.get_object_all_subtypes(self.type)
- return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+ return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
# def build_crypto_regex(subtype, search_id):
@@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol):
return 'zcash'
elif symbol == 'DASH':
return 'dash'
+ elif symbol == 'TRX':
+ return 'tron'
return None
@@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype):
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == '':
pass
- elif subtype == 'name':
- pass
- elif subtype == 'mail':
- pass
return name_to_search
def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py
index f9b09b48..950b2c1c 100755
--- a/bin/lib/objects/UsersAccount.py
+++ b/bin/lib/objects/UsersAccount.py
@@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
def get_last_name(self):
return self._get_field('lastname')
+ def get_name(self):
+ first_name = self.get_first_name()
+ last_name = self.get_last_name()
+ if first_name and last_name:
+ return f'{first_name} {last_name}'
+ elif first_name:
+ return first_name
+ elif last_name:
+ return last_name
+
def get_phone(self):
return self._get_field('phone')
@@ -129,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
def get_messages(self):
messages = []
- for mess in self.get_correlation('message'):
- messages.append(f'message:{mess}')
+ correl = self.get_correlation('message')
+ if 'message' in correl:
+ for mess_id in correl['message']:
+ messages.append(f'message:{mess_id}')
return messages
-
def get_messages_by_chat_obj(self, chat_obj):
messages = []
for mess in self.get_correlation_iter_obj(chat_obj, 'message'):
@@ -157,6 +168,7 @@ class UserAccount(AbstractSubtypeObject):
meta['usernames'] = self.get_usernames()
if 'icon' in options:
meta['icon'] = self.get_icon()
+ meta['svg_icon'] = meta['icon']
if 'info' in options:
meta['info'] = self.get_info()
if 'translation' in options and translation_target:
diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py
index 1a87d1c8..2203ead0 100755
--- a/bin/lib/objects/abstract_object.py
+++ b/bin/lib/objects/abstract_object.py
@@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
-from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
+from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
@@ -299,6 +299,9 @@ class AbstractObject(ABC):
def get_nb_relationships(self, filter=[]):
return get_obj_nb_relationships(self.get_global_id())
+ def get_obj_relationships(self, relationships=set(), filter_types=set()):
+ return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
+
def add_relationship(self, obj2_global_id, relationship, source=True):
# is source
if source:
diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py
index 143979ec..eb6e84d1 100755
--- a/bin/lib/objects/ail_objects.py
+++ b/bin/lib/objects/ail_objects.py
@@ -562,24 +562,47 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
# --- CORRELATION --- #
+#### RELATIONSHIPS ####
+
+def get_relationships():
+ return relationships_engine.get_relationships()
+
+def sanitize_relationships(relationships):
+ return relationships_engine.sanitize_relationships(relationships)
def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_nb_relationships(filter=filter_types)
-def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
objs_hidden=set(),
flask_context=False):
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
- nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
- filter_types=filter_types,
- max_nodes=max_nodes, level=level,
- objs_hidden=objs_hidden)
+ nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
+ filter_types=filter_types,
+ max_nodes=max_nodes, level=level,
+ objs_hidden=objs_hidden)
# print(meta)
meta['objs'] = list(meta['objs'])
return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
"links": links,
"meta": meta}
+def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
+ if obj_type == 'chat':
+ obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+ data = relationships_engine.get_chat_forward_stats(obj_global_id)
+ return data
+ return []
+
+def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
+ if obj_type == 'chat':
+ obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+ data = relationships_engine.get_chat_mentions_stats(obj_global_id)
+ return data
+ return []
+
+# --- RELATIONSHIPS --- #
+
# if __name__ == '__main__':
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py
index 6791214a..653e9c1b 100755
--- a/bin/lib/relationships_engine.py
+++ b/bin/lib/relationships_engine.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
-
+import json
import os
import sys
@@ -16,41 +16,103 @@ config_loader = None
RELATIONSHIPS = {
- "forward",
+ "forwarded_from",
+ "forwarded_to", # forwarded_to
+ "in",
"mention"
}
+
+RELATIONSHIPS_OBJS = { # TODO forward user-account
+ "forwarded_from": {
+ 'chat': {'message'},
+ 'message': {'chat', 'user-account'}
+ },
+ "forwarded_to": {
+ 'chat': {'chat'},
+ 'user-account': {'chat'},
+ },
+ "in": {
+ 'chat': {'message'},
+ 'message': {'chat'}
+ },
+ "mention": {
+ 'chat': {'chat', 'user-account', 'message'},
+ 'message': {'chat', 'user-account'},
+ 'user-account': {'chat', 'message'},
+ },
+}
+
def get_relationships():
return RELATIONSHIPS
-
-def get_obj_relationships_by_type(obj_global_id, relationship):
- return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
-
-def get_obj_nb_relationships_by_type(obj_global_id, relationship):
- return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
-
-def get_obj_relationships(obj_global_id):
- relationships = []
- for relationship in get_relationships():
- for rel in get_obj_relationships_by_type(obj_global_id, relationship):
- meta = {'relationship': relationship}
- direction, obj_id = rel.split(':', 1)
- if direction == 'i':
- meta['source'] = obj_id
- meta['target'] = obj_global_id
- else:
- meta['target'] = obj_id
- meta['source'] = obj_global_id
-
- if not obj_id.startswith('chat'):
- continue
-
- meta['id'] = obj_id
- # meta['direction'] = direction
- relationships.append(meta)
+def sanitize_relationships(relationships):
+ rels = get_relationships()
+ if relationships:
+ relationships = set(relationships).intersection(rels)
+ if not relationships:
+ relationships = rels
+ if not relationships:
+ return []
return relationships
-def get_obj_nb_relationships(obj_global_id):
+def get_relationship_obj_types(relationship):
+ return RELATIONSHIPS_OBJS.get(relationship, {})
+
+def get_relationship_objs(relationship, obj_type):
+ return get_relationship_obj_types(relationship).get(obj_type, set())
+
+def sanityze_obj_types(relationship, obj_type, filter_types):
+ objs_types = get_relationship_objs(relationship, obj_type)
+ if filter_types:
+ filter_types = objs_types.intersection(filter_types)
+ else:
+ filter_types = objs_types
+ # if not filter_types:
+ # filter_types = objs_types
+ # if not filter_types:
+ # return []
+ return filter_types
+
+# TODO check obj_type
+# TODO sanitize relationships
+
+def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+ obj_type = obj_global_id.split(':', 1)[0]
+ relationships = {}
+ filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+ for o_type in filter_types:
+ relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
+ return relationships
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
+ obj_type = obj_global_id.split(':', 1)[0]
+ relationships = {}
+ filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
+ for o_type in filter_types:
+ relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
+ return relationships
+
+def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
+ all_relationships = []
+ for relationship in relationships:
+ obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
+ for obj_type in obj_relationships:
+ for rel in obj_relationships[obj_type]:
+ meta = {'relationship': relationship}
+ direction, obj_id = rel.split(':', 1)
+ if direction == 'i':
+ meta['source'] = obj_id
+ meta['target'] = obj_global_id
+ else:
+ meta['target'] = obj_id
+ meta['source'] = obj_global_id
+
+ meta['id'] = obj_id
+ # meta['direction'] = direction
+ all_relationships.append(meta)
+ return all_relationships
+
+def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
nb = {}
for relationship in get_relationships():
nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
@@ -59,27 +121,27 @@ def get_obj_nb_relationships(obj_global_id):
# TODO Filter by obj type ???
def add_obj_relationship(source, target, relationship):
- r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
- r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
- # r_rel.sadd(f'rels:{source}', relationship)
- # r_rel.sadd(f'rels:{target}', relationship)
+ source_type = source.split(':', 1)[0]
+ target_type = target.split(':', 1)[0]
+ r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
+ r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
-def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
links = []
nodes = set()
meta = {'complete': True, 'objs': set()}
done = set()
done_link = set()
- _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+ _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
return nodes, links, meta
-def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
meta['objs'].add(obj_global_id)
nodes.add(obj_global_id)
- for rel in get_obj_relationships(obj_global_id):
+ for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
meta['objs'].add(rel['id'])
if rel['id'] in done:
@@ -99,13 +161,119 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
if level > 0:
next_level = level - 1
- _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+ _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
# done.add(rel['id'])
+# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
-if __name__ == '__main__':
- source = ''
- target = ''
- add_obj_relationship(source, target, 'forward')
- # print(get_obj_relationships(source))
+def get_chat_forward_stats_out(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
+ chat_mess = rel['source'].split('/')
+
+ chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ if chat_target not in nb:
+ nb[chat_target] = 0
+ nb[chat_target] += 1
+ return nb
+
+def get_chat_forward_stats_in(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+ r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
+ if r:
+ if not r[0]['target'] in nb:
+ nb[r[0]['target']] = 0
+ nb[r[0]['target']] += 1
+ # chat_mess = rel['source'].split('/')
+ #
+ # chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ # print(chat_target, chat, chat_target == chat)
+ # if chat is None or chat_target == chat:
+ # if chat_target not in nb:
+ # nb[chat_target] = 0
+ # nb[chat_target] += 1
+ #
+ # print(json.dumps(nb, indent=4))
+ return nb
+
+def get_chat_forward_stats(obj_global_id): # objs_hidden
+ data = []
+
+ # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+ in_mess = get_chat_forward_stats_out(obj_global_id)
+
+ out_mess = get_chat_forward_stats_in(obj_global_id)
+
+
+ # if rel['target'] == obj_global_id:
+ # # print(rel)
+ # nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
+ #
+ for target in in_mess:
+ data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+ for source in out_mess:
+ data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+ # print()
+ # print(in_mess)
+ # print()
+ # print(out_mess)
+
+ return data
+
+##################################################################
+
+def get_chat_mention_stats_in(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
+ chat_mess = rel['source'].split('/')
+ chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ if chat_source not in nb:
+ nb[chat_source] = 0
+ nb[chat_source] += 1
+ return nb
+
+def get_chat_mention_stats_out(obj_global_id):
+ nb = {}
+ for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+ r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
+ if r:
+ if not r[0]['target'] in nb:
+ nb[r[0]['target']] = 0
+ nb[r[0]['target']] += 1
+ # chat_mess = rel['source'].split('/')
+ #
+ # chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+ # print(chat_target, chat, chat_target == chat)
+ # if chat is None or chat_target == chat:
+ # if chat_target not in nb:
+ # nb[chat_target] = 0
+ # nb[chat_target] += 1
+ #
+ # print(json.dumps(nb, indent=4))
+ return nb
+
+def get_chat_mentions_stats(obj_global_id): # objs_hidden
+ data = []
+
+ # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+ out_mess = get_chat_mention_stats_in(obj_global_id)
+ in_mess = get_chat_mention_stats_out(obj_global_id)
+ #
+ for target in in_mess:
+ data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+ for source in out_mess:
+ data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+ # print()
+ # print(in_mess)
+ # print()
+ # print(out_mess)
+
+ return data
diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py
index 036b02b4..cf36ed81 100755
--- a/bin/modules/Cryptocurrencies.py
+++ b/bin/modules/Cryptocurrencies.py
@@ -92,7 +92,13 @@ CURRENCIES = {
'regex': r'\b(? {
+
+ // Filter data by value between target and source
+ if (min_value > 0) {
+ data.data = data.data.filter(function(value) {
+ return data.value >= min_value;
+ });
+ }
+ if (max_value > 0) {
+ data.data = data.data.filter(function(value) {
+ return data.value <= max_value;
+ });
+ }
+
+
+ function getMaxCharsToShow(angle, radius) {
+ const approximateCharWidth = 7; // Approximate width of a character in pixels
+ const arcLength = angle * radius; // Length of the arc
+ return Math.floor(arcLength / approximateCharWidth); // Maximum number of characters that can fit
+ }
+
+ const width = 900;
+ const height = width;
+ const innerRadius = Math.min(width, height) * 0.5 - 20;
+ const outerRadius = innerRadius + 6;
+
+ const labels_meta = data.meta
+ data = data.data
+
+ // Compute a dense matrix from the weighted links in data.
+ var names = Array.from(d3.union(data.flatMap(d => [d.source, d.target])));
+ const index = new Map(names.map((name, i) => [name, i]));
+ const matrix = Array.from(index, () => new Array(names.length).fill(0));
+ for (const {source, target, value} of data) matrix[index.get(source)][index.get(target)] += value;
+
+ const chord = d3.chordDirected()
+ .padAngle(12 / innerRadius)
+ .sortSubgroups(d3.descending)
+ .sortChords(d3.descending);
+
+ const arc = d3.arc()
+ .innerRadius(innerRadius)
+ .outerRadius(outerRadius);
+
+ const ribbon = d3.ribbonArrow()
+ .radius(innerRadius - 0.5)
+ .padAngle(1 / innerRadius);
+
+ const colors = d3.schemeCategory10;
+
+ const svg = d3.select(container_id)
+ .append("svg")
+ .attr("width", width)
+ .attr("height", height)
+ .attr("viewBox", [-width / 2, -height / 2, width, height])
+ .attr("style", "width: 100%; height: auto; font: 10px sans-serif;");
+
+ const chords = chord(matrix);
+
+ const textId = `text-${Math.random().toString(36).substring(2, 15)}`;
+
+ svg.append("path")
+ .attr("id", textId)
+ .attr("fill", "none")
+ .attr("d", d3.arc()({outerRadius, startAngle: 0, endAngle: 2 * Math.PI}));
+
+ svg.append("g")
+ .attr("fill-opacity", 0.75)
+ .selectAll("path")
+ .data(chords)
+ .enter()
+ .append("path")
+ .attr("d", ribbon)
+ .attr("fill", d => colors[d.target.index])
+ .style("mix-blend-mode", "multiply")
+ .append("title")
+ .text(d => `${labels_meta[names[d.source.index]]}
+=> ${d.source.value} Messages
+${labels_meta[names[d.target.index]]}`);
+
+ const g = svg.append("g")
+ .selectAll("g")
+ .data(chords.groups)
+ .enter()
+ .append("g");
+
+ g.append("path")
+ .attr("d", arc)
+ .attr("fill", d => colors[d.index])
+ .attr("stroke", "#fff")
+ .on("mouseover", function(event, d) {
+ fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]]);
+ })
+ .on("mouseout", fct_mouseout);
+
+ g.each(function(d) {
+ const group = d3.select(this);
+ const angle = d.endAngle - d.startAngle;
+ const text = labels_meta[names[d.index]];
+ const maxCharsToShow = getMaxCharsToShow(angle, outerRadius);
+
+ let displayedText
+ if (maxCharsToShow <= 1) {
+ displayedText = text[0];
+ } else {
+ displayedText = text.length > maxCharsToShow ? text.slice(0, maxCharsToShow - 1) + "…" : text;
+ }
+
+ let info_text = "OUT: " + d3.sum(matrix[d.index]) + " ; IN: " + d3.sum(matrix, row => row[d.index]) + " Messages"
+
+ if (displayedText) {
+ group.append("text")
+ .attr("dy", -3)
+ .append("textPath")
+ .attr("xlink:href", `#${textId}`)
+ .attr("startOffset", d.startAngle * outerRadius)
+ .on("mouseover", function(event, d) {
+ fct_mouseover(event, d, names[d.index], labels_meta[names[d.index]], info_text);
+ })
+ .on("mouseout", fct_mouseout)
+ .text(displayedText);
+ }
+ });
+
+ //return svg.node();
+}
+
+
+
+// return svg ???
+ // return svg
+
diff --git a/var/www/static/js/d3/heatmap_week_hour.js b/var/www/static/js/d3/heatmap_week_hour.js
index 4696a052..97f482be 100644
--- a/var/www/static/js/d3/heatmap_week_hour.js
+++ b/var/www/static/js/d3/heatmap_week_hour.js
@@ -1,4 +1,4 @@
-//Requirement: - D3v5
+//Requirement: - D3v7
// - jquery
// data input: var data = [{"count":0,"date":"2023-11-20","day":0,"hour":0}
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
@@ -33,10 +33,12 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// create a tooltip
- const tooltip = d3.select(container_id)
+ const tooltip = d3.select("body")
.append("div")
+ .attr("class", "tooltip_heatmap")
.style("opacity", 0)
- .attr("class", "tooltip")
+ .style("position", "absolute")
+ .style("pointer-events", "none")
.style("background-color", "white")
.style("border", "solid")
.style("border-width", "2px")
@@ -44,24 +46,28 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.style("padding", "5px")
// Three function that change the tooltip when user hover / move / leave a cell
- const mouseover = function(d) {
- tooltip.style("opacity", 1)
- d3.select(this)
+ const mouseover = function(event, d) {
+ d3.select(event.target)
.style("stroke", "black")
- //.style("stroke-opacity", 1)
+ .style("stroke-opacity", 1)
- var xPosition = d3.mouse(this)[0] + margin.left;
- var yPosition = d3.mouse(this)[1] + margin.top + window.scrollY + 100;
+ let d3_pageX = event.pageX;
+ let d3_pageY = event.pageY;
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: " + d.count + " messages")
- .style("left", xPosition + "px")
- .style("top", yPosition + "px");
+ .style("left", (d3_pageX) + "px")
+ .style("top", (d3_pageY - 28) + "px");
+ tooltip.transition()
+ .duration(200)
+ .style("opacity", 1);
}
- const mouseleave = function(d) {
- tooltip.style("opacity", 0)
- d3.select(this)
+ const mouseleave = function(event, d) {
+ d3.select(event.target)
.style("stroke", "white")
//.style("stroke-opacity", 0.8)
+ tooltip.transition()
+ .duration(200)
+ .style("opacity", 0)
}
///////////////////////////////////////////////////////////////////////////
@@ -114,22 +120,22 @@ const create_heatmap_week_hour = (container_id, data, options) => {
var heatMap = svg.selectAll(".hour")
.data(data)
.enter().append("rect")
- .attr("x", function (d) {
- return (d.hour - 1) * gridSize;
- })
- .attr("y", function (d) {
- return (d.day - 1) * gridSize;
- })
- .attr("class", "hour bordered")
- .attr("width", gridSize)
- .attr("height", gridSize)
- .style("stroke", "white")
- .style("stroke-opacity", 0.6)
- .style("fill", function (d) {
- return colorScale(d.count);
- })
- .on("mouseover", mouseover)
- .on("mouseleave", mouseleave);
+ .attr("x", function (d) {
+ return (d.hour - 1) * gridSize;
+ })
+ .attr("y", function (d) {
+ return (d.day - 1) * gridSize;
+ })
+ .attr("class", "hour bordered")
+ .attr("width", gridSize)
+ .attr("height", gridSize)
+ .style("stroke", "white")
+ .style("stroke-opacity", 0.6)
+ .style("fill", function (d) {
+ return colorScale(d.count);
+ })
+ .on("mouseover", mouseover)
+ .on("mouseleave", mouseleave);
//Append title to the top
svg.append("text")
@@ -221,8 +227,7 @@ const create_heatmap_week_hour = (container_id, data, options) => {
.attr("class", "axis")
.attr("transform", "translate(0," + (10) + ")")
.call(xAxis);
-
-
+
// return svg ???
// return svg
diff --git a/var/www/static/js/helper.js b/var/www/static/js/helper.js
index 3e95484d..0f493bc4 100644
--- a/var/www/static/js/helper.js
+++ b/var/www/static/js/helper.js
@@ -1,4 +1,4 @@
// sanitise str_to_sanitize
function sanitize_text(str_to_sanitize) {
return $("").text(str_to_sanitize).html()
-};
+}
diff --git a/var/www/templates/chats_explorer/basic_card_chat.html b/var/www/templates/chats_explorer/basic_card_chat.html
index d747ff8e..4190788e 100644
--- a/var/www/templates/chats_explorer/basic_card_chat.html
+++ b/var/www/templates/chats_explorer/basic_card_chat.html
@@ -85,7 +85,7 @@
- {{ meta["nb_messages"] }}
+ {{ meta["nb_messages"] }}
{% endif %}
diff --git a/var/www/templates/chats_explorer/card_chat.html b/var/www/templates/chats_explorer/card_chat.html
index 99a0ce38..77227ed4 100644
--- a/var/www/templates/chats_explorer/card_chat.html
+++ b/var/www/templates/chats_explorer/card_chat.html
@@ -95,6 +95,10 @@
{# {{ meta['nb_correlations'] }}#}
+
+
+
{% endif %}
diff --git a/var/www/templates/chats_explorer/card_user_account.html b/var/www/templates/chats_explorer/card_user_account.html
index f5514967..a892c3ca 100644
--- a/var/www/templates/chats_explorer/card_user_account.html
+++ b/var/www/templates/chats_explorer/card_user_account.html
@@ -87,6 +87,11 @@
{% else %}
+
+
+
diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html
index 188779b2..a1832af3 100644
--- a/var/www/templates/chats_explorer/user_account.html
+++ b/var/www/templates/chats_explorer/user_account.html
@@ -12,12 +12,14 @@
+
-
+
+
@@ -34,6 +36,9 @@
{% include 'chats_explorer/card_user_account.html' %}
+
+ {% include 'objects/image/block_blur_img_slider.html' %}
+
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
@@ -42,10 +47,16 @@
User All Messages:
+ Numbers of Messages Posted by Chat:
+
+
+ Numbers of Mentions:
+
+
User Chats:
{% for meta_chats in meta['chats'] %}
- {% with meta=meta_chats %}
+ {% with meta=meta_chats,main_obj_id=meta['id'] %}
{% include 'chats_explorer/basic_card_chat.html' %}
{% endwith %}
@@ -58,6 +69,8 @@
+
+{% include 'objects/tooltip_ail_objects.html' %}
-
-
-