chg: [tags] add Tag class

This commit is contained in:
terrtia 2024-02-02 11:15:08 +01:00
parent e4f21f05cc
commit 45bdc2707b
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 146 additions and 57 deletions

View file

@ -685,7 +685,7 @@ def confirm_tag(tag, obj):
def update_tag_global_by_obj_type(tag, obj_type, subtype=''): def update_tag_global_by_obj_type(tag, obj_type, subtype=''):
tag_deleted = False tag_deleted = False
if obj_type == 'item' or obj_type == 'message': if obj_type == 'item' or obj_type == 'message':
if not r_tags.exists(f'tag_metadata:{tag}'): if not r_tags.exists(f'tag_metadata:{tag}'): # TODO FIXME #################################################################
tag_deleted = True tag_deleted = True
else: else:
if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'): if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'):
@ -1200,12 +1200,17 @@ def get_enabled_tags_with_synonyms_ui():
# TYPE -> taxonomy/galaxy/custom # TYPE -> taxonomy/galaxy/custom
# TODO GET OBJ Types
class Tag: class Tag:
def __int__(self, name: str, local=False): # TODO Get first seen by object, obj='item def __int__(self, name: str, local=False): # TODO Get first seen by object, obj='item
self.name = name self.name = name
self.local = local self.local = local
# TODO
def exists(self):
pass
def is_local(self): def is_local(self):
return self.local return self.local
@ -1216,7 +1221,11 @@ class Tag:
else: else:
return 'taxonomy' return 'taxonomy'
def is_taxonomy(self):
return not self.local and self.is_galaxy()
def is_galaxy(self):
return not self.local and self.name.startswith('misp-galaxy:')
def get_first_seen(self, r_int=False): def get_first_seen(self, r_int=False):
first_seen = r_tags.hget(f'meta:tag:{self.name}', 'first_seen') first_seen = r_tags.hget(f'meta:tag:{self.name}', 'first_seen')
@ -1227,6 +1236,9 @@ class Tag:
first_seen = 99999999 first_seen = 99999999
return first_seen return first_seen
def set_first_seen(self, first_seen):
return r_tags.hget(f'meta:tag:{self.name}', 'first_seen', int(first_seen))
def get_last_seen(self, r_int=False): def get_last_seen(self, r_int=False):
last_seen = r_tags.hget(f'meta:tag:{self.name}', 'last_seen') # 'last_seen:object' -> only if date or daterange last_seen = r_tags.hget(f'meta:tag:{self.name}', 'last_seen') # 'last_seen:object' -> only if date or daterange
if r_int: if r_int:
@ -1236,6 +1248,9 @@ class Tag:
last_seen = 0 last_seen = 0
return last_seen return last_seen
def set_last_seen(self, last_seen):
return r_tags.hset(f'meta:tag:{self.name}', 'last_seen', int(last_seen)
def get_color(self): def get_color(self):
color = r_tags.hget(f'meta:tag:{self.name}', 'color') color = r_tags.hget(f'meta:tag:{self.name}', 'color')
if not color: if not color:
@ -1258,6 +1273,131 @@ class Tag:
'local': self.is_local()} 'local': self.is_local()}
return meta return meta
def update_obj_type_first_seen(self, obj_type, first_seen, last_seen): # TODO SUBTYPE ##################################
if int(first_seen) > int(last_seen):
raise Exception(f'INVALID first_seen/last_seen, {first_seen}/{last_seen}')
for date in Date.get_daterange(first_seen, last_seen):
date = int(date)
if date == last_seen:
if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') > 0:
r_tags.hset(f'tag_metadata:{self.name}', 'first_seen', first_seen)
else:
r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen') # TODO SUBTYPE
r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen') # TODO SUBTYPE
r_tags.srem(f'list_tags:{obj_type}', self.name) # TODO SUBTYPE
elif r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') > 0:
r_tags.hset(f'tag_metadata:{self.name}', 'first_seen', first_seen) # TODO METADATA OBJECT NAME
def update_obj_type_last_seen(self, obj_type, first_seen, last_seen): # TODO SUBTYPE ##################################
if int(first_seen) > int(last_seen):
raise Exception(f'INVALID first_seen/last_seen, {first_seen}/{last_seen}')
for date in Date.get_daterange(first_seen, last_seen).reverse():
date = int(date)
if date == last_seen:
if r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') > 0:
r_tags.hset(f'tag_metadata:{self.name}', 'last_seen', last_seen)
else:
r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen') # TODO SUBTYPE
r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen') # TODO SUBTYPE
r_tags.srem(f'list_tags:{obj_type}', self.name) # TODO SUBTYPE
elif r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') > 0:
r_tags.hset(f'tag_metadata:{self.name}', 'last_seen', last_seen) # TODO METADATA OBJECT NAME
# TODO
# TODO Update First seen and last seen
# TODO SUBTYPE CHATS ??????????????
def update_obj_type_date(self, obj_type, date, op='add', first_seen=None, last_seen=None):
date = int(date)
if not first_seen:
first_seen = self.get_first_seen(r_int=True)
if not last_seen:
last_seen = self.get_last_seen(r_int=True)
# Add tag
if op == 'add':
if date < first_seen:
self.set_first_seen(date)
if date > last_seen:
self.set_last_seen(date)
# Delete tag
else:
if date == first_seen and date == last_seen:
# TODO OBJECTS ##############################################################################################
if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') < 1: ####################### TODO OBJ SUBTYPE ???????????????????
r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen')
r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen')
# TODO CHECK IF DELETE FULL TAG LIST ############################
elif date == first_seen:
if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') < 1:
if int(last_seen) >= int(first_seen):
self.update_obj_type_first_seen(obj_type, first_seen, last_seen) # TODO OBJ_TYPE
elif date == last_seen:
if r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') < 1:
if int(last_seen) >= int(first_seen):
self.update_obj_type_last_seen(obj_type, first_seen, last_seen) # TODO OBJ_TYPE
# STATS
nb = r_tags.hincrby(f'daily_tags:{date}', self.name, -1)
if nb < 1:
r_tags.hdel(f'daily_tags:{date}', self.name)
# TODO -> CHECK IF TAG EXISTS + UPDATE FIRST SEEN/LAST SEEN
def update(self, date=None):
pass
# TODO CHANGE ME TO SUB FUNCTION ##### add_object_tag(tag, obj_type, obj_id, subtype='')
def add(self, obj_type, subtype, obj_id):
if subtype is None:
subtype = ''
if r_tags.sadd(f'tag:{obj_type}:{subtype}:{obj_id}', self.name) == 1:
r_tags.sadd('list_tags', self.name)
r_tags.sadd(f'list_tags:{obj_type}', self.name)
if subtype:
r_tags.sadd(f'list_tags:{obj_type}:{subtype}', self.name)
if obj_type == 'item':
date = item_basic.get_item_date(obj_id)
# add domain tag
if item_basic.is_crawled(obj_id) and self.name != 'infoleak:submission="crawler"' and self.name != 'infoleak:submission="manual"':
domain = item_basic.get_item_domain(obj_id)
self.add('domain', '', domain)
elif obj_type == 'message':
timestamp = obj_id.split('/')[1]
date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
else:
date = None
if date:
r_tags.sadd(f'{obj_type}:{subtype}:{self.name}:{date}', obj_id)
update_tag_metadata(self.name, date)
else:
r_tags.sadd(f'{obj_type}:{subtype}:{self.name}', obj_id)
# TODO REPLACE ME BY DATE TAGS ????
# STATS BY TYPE ???
# DAILY STATS
r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', self.name, 1)
# TODO CREATE FUNCTION GET OBJECT DATE
def remove(self, obj_type, subtype, obj_id):
# TODO CHECK IN ALL OBJECT TO DELETE
pass
def delete(self):
pass
#### TAG AUTO PUSH #### #### TAG AUTO PUSH ####

View file

@ -128,7 +128,10 @@ class UserAccount(AbstractSubtypeObject):
self._get_timeline_username().add_timestamp(timestamp, username_global_id) self._get_timeline_username().add_timestamp(timestamp, username_global_id)
def get_messages_by_chat_obj(self, chat_obj): def get_messages_by_chat_obj(self, chat_obj):
return self.get_correlation_iter_obj(chat_obj, 'message') messages = []
for mess in self.get_correlation_iter_obj(chat_obj, 'message'):
messages.append(f'message:{mess}')
return messages
def get_meta(self, options=set(), translation_target=None): # TODO Username timeline def get_meta(self, options=set(), translation_target=None): # TODO Username timeline
meta = self._get_meta(options=options) meta = self._get_meta(options=options)

View file

@ -31,61 +31,7 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
{# <div class="card my-3">#} TODO CHAT abstract metadata <h5>{{ meta['type'] }} {{ meta['subtype'] }} {{ meta['id'] }}</h5>
{##}
{# <div class="card-header" style="background-color:#d9edf7;font-size: 15px">#}
{# <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>#}
{# {% if chat['icon'] %}#}
{# <div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>#}
{# {% endif %}#}
{# <ul class="list-group mb-2">#}
{# <li class="list-group-item py-0">#}
{# <table class="table">#}
{# <thead class="">#}
{# <tr>#}
{# <th>Name</th>#}
{# <th>ID</th>#}
{# <th>Created at</th>#}
{# <th>First Seen</th>#}
{# <th>Last Seen</th>#}
{# <th>NB Sub-Channels</th>#}
{# <th>Participants</th>#}
{# </tr>#}
{# </thead>#}
{# <tbody style="font-size: 15px;">#}
{# <tr>#}
{# <td>{{ chat['name'] }}</td>#}
{# <td>{{ chat['id'] }}</td>#}
{# <td>{{ chat['created_at'] }}</td>#}
{# <td>#}
{# {% if chat['first_seen'] %}#}
{# {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>#}
{# {% if chat['last_seen'] %}#}
{# {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>{{ chat['nb_subchannels'] }}</td>#}
{# <td>#}
{# <a href="{{ url_for('chats_explorer.objects_thread_messages')}}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}"><i class="far fa-user-circle"></i> {{ chat['participants'] | length}}</a>#}
{##}
{# </td>#}
{# </tr>#}
{# </tbody>#}
{# </table>#}
{# {% if chat['info'] %}#}
{# <li class="list-group-item py-0">#}
{# <pre class="my-0">{{ chat['info'] }}</pre>#}
{# </li>#}
{# {% endif %}#}
{# </li>#}
{# </ul>#}
{##}
{# </div>#}
{# </div>#}
<h4>Participants:</h4> <h4>Participants:</h4>
<table id="tableparticipantss" class="table"> <table id="tableparticipantss" class="table">