From 800098540bd7b534702abc675e767bc6be8fd7e3 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 11 Mar 2024 15:33:30 +0100 Subject: [PATCH 01/15] chg: [correlation] description: get message content + don't show etag, cookie_name and hhash by default --- bin/lib/ail_core.py | 8 ++++++++ bin/lib/objects/Images.py | 5 ++++- bin/lib/objects/ail_objects.py | 9 ++++++--- var/www/blueprints/correlation.py | 7 +++++-- var/www/templates/correlation/show_correlation.html | 4 +++- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 029af5fb..44cd1359 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -22,6 +22,11 @@ AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', ' AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} +# TODO by object TYPE ???? +AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded', + 'domain', 'favicon', 'file-name', + 'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) + def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') if not ail_uuid: @@ -69,6 +74,9 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype return r_object.smembers(f'all_chat:subtypes') return [] +def get_default_correlation_objects(): + return AIL_OBJECTS_CORRELATIONS_DEFAULT + def get_obj_queued(): return ['item', 'image'] diff --git a/bin/lib/objects/Images.py b/bin/lib/objects/Images.py index 280ac06c..9d71b7d5 100755 --- a/bin/lib/objects/Images.py +++ b/bin/lib/objects/Images.py @@ -71,7 +71,10 @@ class Image(AbstractDaterangeObject): return file_content def get_content(self, r_type='str'): - return self.get_file_content() + if r_type == 'str': + return None + else: + return self.get_file_content() def get_misp_object(self): obj_attrs = [] diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 5c083e0f..33413943 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -12,7 +12,7 @@ from lib.exceptions import AILObjectUnknown from lib.ConfigLoader import ConfigLoader -from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes +from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes, get_default_correlation_objects from lib import correlations_engine from lib import relationships_engine from lib import btc_ail @@ -53,13 +53,16 @@ def is_object_subtype(obj_type): def is_valid_object_subtype(obj_type, subtype): return subtype in get_object_all_subtypes(obj_type) -def sanitize_objs_types(objs): +def sanitize_objs_types(objs, default=False): l_types = [] for obj in objs: if is_valid_object_type(obj): l_types.append(obj) if not l_types: - l_types = get_all_objects() + if default: + l_types = get_default_correlation_objects() + else: + l_types = get_all_objects() return l_types #### OBJECT #### diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 9cd21b24..a5edf49e 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -165,7 +165,7 @@ def show_correlation(): related_btc = bool(request.args.get('related_btc', False)) - filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True) # check if obj_id exist if not ail_objects.exists_obj(obj_type, subtype, obj_id): @@ -206,7 +206,10 @@ def get_description(): return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 # object exist else: - res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'}, + options = {'icon', 'tags', 'tags_safe'} + if obj_type == 'message': + options.add('content') + res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=True) if 'tags' in res: res['tags'] = list(res['tags']) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 04c4ee0b..49c8c90c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -714,7 +714,9 @@ if (d.popover) { } desc = desc + "" } else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") { - desc = desc + "
" + sanitize_text(key) + "
" + sanitize_text(data[key]) + "
" + if (data[key]) { + desc = desc + "
" + sanitize_text(key) + "
" + sanitize_text(data[key]) + "
" + } } }); desc = desc + "" From d526b2fd98d3adc9e6da053d07ef7a90540a9c94 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 11 Mar 2024 15:36:24 +0100 Subject: [PATCH 02/15] chg: [correlation UI] change Direct Correlations position --- .../correlation/show_correlation.html | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 49c8c90c..a3fae335 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -204,6 +204,22 @@
+
    +
  • Direct Correlations
    • +
  • + {% for obj_type in dict_object['nb_correl'] %} +
    +
    + {{ obj_type }} +
    +
    + {{ dict_object['nb_correl'][obj_type] }} +
    +
    + {% endfor %} +
    • +
+
  • Select Correlation
    • @@ -343,21 +359,6 @@

-
    -
  • Direct Correlations
    • -
  • - {% for obj_type in dict_object['nb_correl'] %} -
    -
    - {{ obj_type }} -
    -
    - {{ dict_object['nb_correl'][obj_type] }} -
    -
    - {% endfor %} -
    • -
From 9031376b50092adf48059f580598aa9ada305523 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Mar 2024 13:43:19 +0100 Subject: [PATCH 03/15] fix: [FeederImporter] add all objects in queue --- bin/importer/FeederImporter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 34ccca8c..8532f488 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -103,14 +103,16 @@ class FeederImporter(AbstractImporter): if data_obj: objs.add(data_obj) + objs_messages = [] for obj in objs: if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() - return obj, f'{feeder_name} {gzip64_content}' + relay_message = f'{feeder_name} {gzip64_content}' + objs_messages.append({'obj': obj, 'message': relay_message}) else: # Messages save on DB if obj.exists() and obj.type != 'chat': - return obj, f'{feeder_name}' - + objs_messages.append({'obj': obj, 'message': feeder_name}) + return objs_messages class FeederModuleImporter(AbstractModule): def __init__(self): @@ -128,10 +130,8 @@ class FeederModuleImporter(AbstractModule): def compute(self, message): # TODO HANDLE Invalid JSON json_data = json.loads(message) - # TODO multiple objs + messages - obj, relay_message = self.importer.importer(json_data) - #### - self.add_message_to_queue(obj=obj, message=relay_message) + for obj_message in self.importer.importer(json_data): + self.add_message_to_queue(obj=obj_message['obj'], message=obj_message['message']) # Launch Importer From dc0545dfd072facaa25db440f3f9fe85634d237d Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Mar 2024 13:54:56 +0100 Subject: [PATCH 04/15] chg: [RetroHunt] retro hunt on messages --- bin/lib/chats_viewer.py | 57 +++++++++++++++++++ bin/lib/objects/Messages.py | 4 +- bin/lib/objects/ail_objects.py | 11 +++- bin/trackers/Retro_Hunt.py | 3 + .../templates/hunter/add_retro_hunt_task.html | 4 ++ 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 0ec42772..7f14cb45 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -323,6 +323,63 @@ def get_username_meta_from_global_id(username_global_id): username = Usernames.Username(username_id, instance_uuid) return username.get_meta() + +# TODO Filter +## Instance type +## Chats IDS +## SubChats IDS +## Threads IDS +## Daterange +def get_messages_iterator(filters={}): + + for instance_uuid in get_chat_service_instances(): + + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + messages, _ = subchannel._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads + + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + _, _ = thread._get_messages(nb=-1) + for mess in messages: + message_id, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + + # messages + messages, _ = chat._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads ??? + +def get_nb_messages_iterator(filters={}): + nb_messages = 0 + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + nb_messages += subchannel.get_nb_messages() + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + nb_messages += thread.get_nb_messages() + # messages + nb_messages += chat.get_nb_messages() + return nb_messages + #### API #### def api_get_chat_service_instance(chat_instance_uuid): diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 56281e69..09645cf3 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -85,7 +85,8 @@ class Message(AbstractObject): if r_type == 'str': return content elif r_type == 'bytes': - return content.encode() + if content: + return content.encode() def get_date(self): timestamp = self.get_timestamp() @@ -339,7 +340,6 @@ def create(obj_id, content, translation=None, tags=[]): message.create(content, translation=translation, tags=tags) return message - # TODO Encode translation diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 33413943..63248a79 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -18,6 +18,8 @@ from lib import relationships_engine from lib import btc_ail from lib import Tag +from lib import chats_viewer + from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads @@ -32,7 +34,7 @@ from lib.objects import FilesNames from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Images -from lib.objects.Messages import Message +from lib.objects import Messages from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles @@ -90,7 +92,7 @@ def get_object(obj_type, subtype, obj_id): elif obj_type == 'image': return Images.Image(obj_id) elif obj_type == 'message': - return Message(obj_id) + return Messages.Message(obj_id) elif obj_type == 'screenshot': return Screenshot(obj_id) elif obj_type == 'title': @@ -296,6 +298,9 @@ def obj_iterator(obj_type, filters): return get_all_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.get_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_messages_iterator(filters=filters) + def card_objs_iterators(filters): nb = 0 @@ -310,6 +315,8 @@ def card_obj_iterator(obj_type, filters): return get_nb_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.nb_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_nb_messages_iterator(filters=filters) def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index f6854781..6830a663 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -88,6 +88,9 @@ class Retro_Hunt_Module(AbstractModule): for obj in ail_objects.obj_iterator(obj_type, filters): self.obj = obj content = obj.get_content(r_type='bytes') + if not content: + continue + rule.match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) diff --git a/var/www/templates/hunter/add_retro_hunt_task.html b/var/www/templates/hunter/add_retro_hunt_task.html index d8763f61..fada045e 100644 --- a/var/www/templates/hunter/add_retro_hunt_task.html +++ b/var/www/templates/hunter/add_retro_hunt_task.html @@ -78,6 +78,10 @@ +
+ + +
{#
#} {# #} {# #} From 27b2679ba6a9058721a57a8ac66dd13ddcb39a59 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 13 Mar 2024 10:07:11 +0100 Subject: [PATCH 05/15] fix: [tracker] fix yara content error on empty message --- bin/trackers/Tracker_Yara.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 344c77a7..ebad8b05 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -62,6 +62,8 @@ class Tracker_Yara(AbstractModule): return None content = self.obj.get_content(r_type='bytes') + if not content: + return None try: yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, From eb6adc4b98c3d15da8a5e7e947a81c8ac38ff29d Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 13 Mar 2024 10:11:03 +0100 Subject: [PATCH 06/15] chg: [modules] improve error output --- bin/modules/abstract_module.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index affd6c6a..273f7580 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -179,7 +179,10 @@ class AbstractModule(ABC): trace = traceback.format_tb(err.__traceback__) trace = ''.join(trace) self.logger.critical(f"Error in module {self.module_name}: {__name__} : {err}") - self.logger.critical(f"Module {self.module_name} input message: {message}") + if message: + self.logger.critical(f"Module {self.module_name} input message: {message}") + if self.obj: + self.logger.critical(f"{self.module_name} Obj: {self.obj.get_global_id()}") self.logger.critical(trace) if isinstance(err, ModuleQueueError): From 13372f8c85baad0e0b6417b65bb34a3cc2e99263 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 13 Mar 2024 11:58:40 +0100 Subject: [PATCH 07/15] fix: [dashboard] fix objects links --- bin/lib/telegram.py | 7 ++--- bin/modules/ApiKey.py | 6 ++--- bin/modules/Credential.py | 4 +-- bin/modules/CreditCards.py | 4 +-- bin/modules/Cryptocurrencies.py | 2 +- bin/modules/CveModule.py | 2 +- bin/modules/DomClassifier.py | 6 ++--- bin/modules/Duplicates.py | 4 +-- bin/modules/IPAddress.py | 4 +-- bin/modules/Iban.py | 2 +- bin/modules/Keys.py | 24 ++++++++--------- bin/modules/LibInjection.py | 2 +- bin/modules/Mail.py | 2 +- bin/modules/Onion.py | 4 +-- bin/modules/Phone.py | 2 +- bin/modules/SQLInjectionDetection.py | 2 +- bin/modules/Tags.py | 2 +- bin/modules/Telegram.py | 2 +- bin/modules/Urls.py | 2 +- bin/trackers/Tracker_Regex.py | 4 +-- bin/trackers/Tracker_Term.py | 6 ++--- bin/trackers/Tracker_Typo_Squatting.py | 4 +-- bin/trackers/Tracker_Yara.py | 4 +-- var/www/blueprints/investigations_b.py | 8 ++++++ var/www/modules/dashboard/Flask_dashboard.py | 3 ++- .../modules/dashboard/templates/index.html | 4 ++- var/www/static/js/indexjavascript.js | 26 ++++++++++--------- 27 files changed, 76 insertions(+), 66 deletions(-) diff --git a/bin/lib/telegram.py b/bin/lib/telegram.py index aca08d38..02984066 100755 --- a/bin/lib/telegram.py +++ b/bin/lib/telegram.py @@ -22,11 +22,8 @@ REGEX_JOIN_HASH = re.compile(r'[0-9a-zA-z-]+') ## ## -def save_item_correlation(username, item_id, item_date): - Username.save_item_correlation('telegram', username, item_id, item_date) - -def save_telegram_invite_hash(invite_hash, item_id): - r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}') +def save_telegram_invite_hash(invite_hash, obj_global_id): + r_obj.sadd('telegram:invite_code', f'{invite_hash};{obj_global_id}') def get_data_from_telegram_url(base_url, url_path): dict_url = {} diff --git a/bin/modules/ApiKey.py b/bin/modules/ApiKey.py index bf54c095..2a9d11f8 100755 --- a/bin/modules/ApiKey.py +++ b/bin/modules/ApiKey.py @@ -61,7 +61,7 @@ class ApiKey(AbstractModule): if google_api_key: print(f'found google api key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="google-api-key"' self.add_message_to_queue(message=tag, queue='Tags') @@ -69,10 +69,10 @@ class ApiKey(AbstractModule): # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY if aws_access_key: print(f'found AWS key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}') if aws_secret_key: print(f'found AWS secret key') - self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="aws-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Credential.py b/bin/modules/Credential.py index f20f2ba9..6d0df6a7 100755 --- a/bin/modules/Credential.py +++ b/bin/modules/Credential.py @@ -103,11 +103,11 @@ class Credential(AbstractModule): print(message) - to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{item.get_id()}' + to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}' # num of creds above threshold, publish an alert if nb_cred > self.criticalNumberToAlert: - print(f"========> Found more than 10 credentials in this file : {item.get_id()}") + print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}") self.redis_logger.warning(to_print) tag = 'infoleak:automatic-detection="credential"' diff --git a/bin/modules/CreditCards.py b/bin/modules/CreditCards.py index 1d8411bb..c237bf05 100755 --- a/bin/modules/CreditCards.py +++ b/bin/modules/CreditCards.py @@ -86,7 +86,7 @@ class CreditCards(AbstractModule): # print(creditcard_set) to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};' if creditcard_set: - mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{item.id}' + mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}' print(mess) self.redis_logger.warning(mess) @@ -96,7 +96,7 @@ class CreditCards(AbstractModule): if r_result: return creditcard_set else: - self.redis_logger.info(f'{to_print}CreditCard related;{item.id}') + self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}') if __name__ == '__main__': diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 5a83689f..036b02b4 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -149,7 +149,7 @@ class Cryptocurrencies(AbstractModule, ABC): item.get_date(), item.get_basename()) self.redis_logger.warning('{}Detected {} {} private key;{}'.format( - to_print, len(private_keys), currency['name'], item_id)) + to_print, len(private_keys), currency['name'], self.obj.get_global_id())) else: private_keys = [] diff --git a/bin/modules/CveModule.py b/bin/modules/CveModule.py index 6904ee28..1387686b 100755 --- a/bin/modules/CveModule.py +++ b/bin/modules/CveModule.py @@ -56,7 +56,7 @@ class CveModule(AbstractModule): cve = Cves.Cve(cve_id) cve.add(date, item) - warning = f'{item_id} contains CVEs {cves}' + warning = f'{self.obj.get_global_id()} contains CVEs {cves}' print(warning) self.redis_logger.warning(warning) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index b4620ee2..fa0a232f 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -82,20 +82,20 @@ class DomClassifier(AbstractModule): localizeddomains = self.dom_classifier.include(expression=self.cc_tld) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}") if self.cc: localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}") if r_result: return self.dom_classifier.vdomain except IOError as err: self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") - raise Exception(f"CRC Checksum Failed on: {item.get_id()}") + raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}") if __name__ == "__main__": diff --git a/bin/modules/Duplicates.py b/bin/modules/Duplicates.py index 57641979..edee6025 100755 --- a/bin/modules/Duplicates.py +++ b/bin/modules/Duplicates.py @@ -92,10 +92,10 @@ class Duplicates(AbstractModule): Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id()) if nb_duplicates: - self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{item.get_id()}') + self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}') y = time.time() - print(f'{item.get_id()} Processed in {y-x} sec') + print(f'{self.obj.get_global_id()} Processed in {y-x} sec') # self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x)) diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 31a0ff68..8899ca43 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -82,8 +82,8 @@ class IPAddress(AbstractModule): matching_ips.append(address) if len(matching_ips) > 0: - self.logger.info(f'{item.get_id()} contains {len(matching_ips)} IPs') - self.redis_logger.warning(f'{item.get_id()} contains {item.get_id()} IPs') + self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs') + self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs') # Tag message with IP tag = 'infoleak:automatic-detection="ip"' diff --git a/bin/modules/Iban.py b/bin/modules/Iban.py index aa1aa5d6..7b0c66d0 100755 --- a/bin/modules/Iban.py +++ b/bin/modules/Iban.py @@ -95,7 +95,7 @@ class Iban(AbstractModule): # Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1) to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};' - self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}') + self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}') # Tags tag = 'infoleak:automatic-detection="iban"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Keys.py b/bin/modules/Keys.py index a2e7288d..2fdb96f8 100755 --- a/bin/modules/Keys.py +++ b/bin/modules/Keys.py @@ -63,7 +63,7 @@ class Keys(AbstractModule): get_pgp_content = False if KeyEnum.PGP_MESSAGE.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a PGP enc message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message') tag = 'infoleak:automatic-detection="pgp-message"' self.add_message_to_queue(message=tag, queue='Tags') @@ -81,21 +81,21 @@ class Keys(AbstractModule): get_pgp_content = True if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message') tag = 'infoleak:automatic-detection="pgp-private-key"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.CERTIFICATE.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a certificate message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message') tag = 'infoleak:automatic-detection="certificate"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.RSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message') print('rsa private key message found') tag = 'infoleak:automatic-detection="rsa-private-key"' @@ -103,7 +103,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message') print('private key message found') tag = 'infoleak:automatic-detection="private-key"' @@ -111,7 +111,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message') print('encrypted private key message found') tag = 'infoleak:automatic-detection="encrypted-private-key"' @@ -119,7 +119,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print('openssh private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -127,7 +127,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message') print('SSH2 private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -135,7 +135,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print('OpenVPN Static key message found') tag = 'infoleak:automatic-detection="vpn-static-key"' @@ -143,21 +143,21 @@ class Keys(AbstractModule): # find = True if KeyEnum.DSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message') tag = 'infoleak:automatic-detection="dsa-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.EC_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an ec private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message') tag = 'infoleak:automatic-detection="ec-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PUBLIC_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a public key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message') tag = 'infoleak:automatic-detection="public-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index de4d1287..fc7a5956 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -70,7 +70,7 @@ class LibInjection(AbstractModule): print(f"Detected (libinjection) SQL in URL: {item_id}") print(unquote(url)) - to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' + to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' self.redis_logger.warning(to_print) # Add tag diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index 82cac546..8a3a66a4 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -172,7 +172,7 @@ class Mail(AbstractModule): # for tld in mx_tlds: # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) - msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item.id}' + msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}' if num_valid_email > self.mail_threshold: print(f'{item.id} Checked {num_valid_email} e-mail(s)') self.redis_logger.warning(msg) diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 681bae0c..34e50247 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -98,8 +98,8 @@ class Onion(AbstractModule): print(f'{domain} added to crawler queue: {task_uuid}') else: to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};' - print(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') - self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') + print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') + self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') # TAG Item tag = 'infoleak:automatic-detection="onion"' diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index 0c01a40f..8eae16cc 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -62,7 +62,7 @@ class Phone(AbstractModule): tag = 'infoleak:automatic-detection="phone-number"' self.add_message_to_queue(message=tag, queue='Tags') - self.redis_logger.warning(f'{item.get_id()} contains {len(phone)} Phone numbers') + self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers') # # List of the regex results in the Item, may be null # results = self.REG_PHONE.findall(content) diff --git a/bin/modules/SQLInjectionDetection.py b/bin/modules/SQLInjectionDetection.py index 7c1bcb4f..953fe82e 100755 --- a/bin/modules/SQLInjectionDetection.py +++ b/bin/modules/SQLInjectionDetection.py @@ -53,7 +53,7 @@ class SQLInjectionDetection(AbstractModule): print(f"Detected SQL in URL: {item.id}") print(urllib.request.unquote(url)) - to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item.id}' + to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' self.redis_logger.warning(to_print) # Tag diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 33ea1c80..0f613eae 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -41,7 +41,7 @@ class Tags(AbstractModule): # Create a new tag item.add_tag(tag) - print(f'{item.get_id()}: Tagged {tag}') + print(f'{self.obj.get_global_id()}: Tagged {tag}') # Forward message to channel self.add_message_to_queue(message=tag, queue='Tag_feed') diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 140948c2..0cba6ef5 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -62,7 +62,7 @@ class Telegram(AbstractModule): print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: - telegram.save_telegram_invite_hash(invite_hash, item.id) + telegram.save_telegram_invite_hash(invite_hash, self.obj.get_global_id()) print(f'invite code: {invite_hash}') invite_code_found = True diff --git a/bin/modules/Urls.py b/bin/modules/Urls.py index c5a3dfe4..ed510a84 100755 --- a/bin/modules/Urls.py +++ b/bin/modules/Urls.py @@ -78,7 +78,7 @@ class Urls(AbstractModule): except AttributeError: url = url_decoded['url'] - print(url, item.get_id()) + print(url, self.obj.get_global_id()) self.add_message_to_queue(message=str(url), queue='Url') self.logger.debug(f"url_parsed: {url}") diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 43fa5764..c23d1589 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -116,8 +116,8 @@ class Tracker_Regex(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked regex found: {tracker_name} in {obj_id}') - self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}') + print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index a3067674..2d0a02a5 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -93,7 +93,7 @@ class Tracker_Term(AbstractModule): try: dict_words_freq = Tracker.get_text_word_frequency(content) except TimeoutException: - self.redis_logger.warning(f"{obj.get_id()} processing timeout") + self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout") else: signal.alarm(0) @@ -124,8 +124,8 @@ class Tracker_Term(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked term {tracker_uuid} found: {tracker_name} in {obj_id}') - self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}') + print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(), obj_id) diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index 9e093b3e..d81d430f 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -75,8 +75,8 @@ class Tracker_Typo_Squatting(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked typosquatting found: {tracked} in {obj_id}') - self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}') + print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index ebad8b05..29542553 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -69,8 +69,8 @@ class Tracker_Yara(AbstractModule): yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60) if yara_match: - self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}') - print(f'{self.obj.get_id()}: {yara_match}') + self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}') + print(f'{self.obj.get_global_id()}: {yara_match}') except yara.TimeoutError: print(f'{self.obj.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py index cf3cf688..70764179 100644 --- a/var/www/blueprints/investigations_b.py +++ b/var/www/blueprints/investigations_b.py @@ -209,6 +209,14 @@ def unregister_investigation(): def get_investigations_selector_json(): return jsonify(Investigations.get_investigations_selector()) +@investigations_b.route("/object/gid") +@login_required +@login_read_only +def get_object_gid(): + obj_global_id = request.args.get('gid') + ail_obj = ail_objects.get_obj_from_global_id(obj_global_id) + url = ail_obj.get_link(flask_context=True) + return redirect(url) # # @investigations_b.route("/object/item") #completely shows the paste in a new tab diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 3f0df03f..64b842fe 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -44,6 +44,7 @@ def event_stream(): pubsub.psubscribe("Script" + '.*') for msg in pubsub.listen(): + # print(msg) type = msg['type'] pattern = msg['pattern'] channel = msg['channel'] @@ -77,7 +78,7 @@ def dashboard_alert(log): log = log[46:].split(';') if len(log) == 6: date_time = datetime_from_utc_to_local(utc_str) - path = url_for('objects_item.showItem', id=log[5]) + path = url_for('investigations_b.get_object_gid', gid=log[5]) res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], 'paste': log[3], 'message': log[4], 'path': path} diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index db5b4fb4..1ac1577b 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -162,7 +162,7 @@ - + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + + +
+
+

{{ meta["id"] }} :

+
    +
  • + + + + + + + + + + + + + + + + + +
    First SeenLast SeenNB
    + + + + {{ meta["icon"]["icon"] }} + + + {{ meta['type'] }} + + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {{ meta['nb_seen'] }}
    +
    • +
  • +
    +
    +
    + +
    +
    +
    + {% include 'objects/image/block_blur_img_slider.html' %} + +
    +
    +
    +
    +
    • + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='image', obj_id=meta['id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index a3fae335..a4286471 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -121,6 +121,8 @@ {% include 'correlation/metadata_card_etag.html' %} {% elif dict_object["object_type"] == "hhhash" %} {% include 'correlation/metadata_card_hhhash.html' %} + {% elif dict_object["object_type"] == "image" %} + {% include 'chats_explorer/card_image.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% elif dict_object["object_type"] == "favicon" %} diff --git a/var/www/templates/domains/card_img_domain.html b/var/www/templates/domains/card_img_domain.html index 66447a72..b9eecfd3 100644 --- a/var/www/templates/domains/card_img_domain.html +++ b/var/www/templates/domains/card_img_domain.html @@ -55,7 +55,7 @@ - {{dict_domain["first_seen"]}} + {{dict_domain["last_seen"]}} diff --git a/var/www/templates/objects/block_object_footer_small.html b/var/www/templates/objects/block_object_footer_small.html new file mode 100644 index 00000000..670c38ca --- /dev/null +++ b/var/www/templates/objects/block_object_footer_small.html @@ -0,0 +1,12 @@ +
+{#
#} +{# #} +{# #} +{# #} +{#
#} +
+ + + +
+
From 599f3ca9534fca89dff00be717debffac055636b Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 18 Mar 2024 15:53:20 +0100 Subject: [PATCH 11/15] chg: [chat] add chat default + basic card --- bin/lib/Tag.py | 4 +- bin/lib/objects/Chats.py | 2 +- bin/lib/objects/ail_objects.py | 5 +- var/www/blueprints/chats_explorer.py | 4 +- var/www/blueprints/tags_ui.py | 27 ++++-- .../chats_explorer/basic_card_chat.html | 67 ++++++++++++++ .../templates/chats_explorer/card_chat.html | 88 +++++++++++++++++++ .../templates/chats_explorer/card_image.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 64 +------------- .../correlation/show_correlation.html | 2 + var/www/templates/modals/edit_tag.html | 2 +- 11 files changed, 189 insertions(+), 78 deletions(-) create mode 100644 var/www/templates/chats_explorer/basic_card_chat.html create mode 100644 var/www/templates/chats_explorer/card_chat.html diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 1dacbfe8..15edcd0a 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1558,14 +1558,14 @@ def get_obj_date(object_type, object_id): return None # API QUERY -def api_delete_obj_tags(tags=[], object_id=None, object_type="item"): +def api_delete_obj_tags(tags=[], object_id=None, object_type="item", subtype=''): if not object_id: return ({'status': 'error', 'reason': 'object id not found'}, 404) if not tags: return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400) for tag in tags: - res = delete_object_tag(tag, object_type, object_id, subtype='') + res = delete_object_tag(tag, object_type, object_id, subtype=subtype) if res: return res diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index dde776b0..0e6dadd0 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -51,7 +51,7 @@ class Chat(AbstractChatObject): def get_link(self, flask_context=False): if flask_context: - url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + url = url_for('chats_explorer.chats_explorer_chat', subtype=self.subtype, id=self.id) else: url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' return url diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 63248a79..14a035a4 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -254,8 +254,9 @@ def get_objects_meta(objs, options=set(), flask_context=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) - meta = obj.get_meta() - meta['icon'] = obj.get_svg_icon() + meta = obj.get_meta(options={'icon', 'info', 'nb_participants'}) + # meta['icon'] = obj.get_svg_icon() + meta['svg_icon'] = obj.get_svg_icon() if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index e24abe82..3363a29b 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -92,7 +92,9 @@ def chats_explorer_chat(): else: chat = chat[0] languages = Language.get_translation_languages() - return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) + return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(chat['id'], chat['type'], chat['subtype']), + translation_languages=languages, translation_target=target) @chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET']) @login_required diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index f833ab16..2bf7faef 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -170,7 +170,11 @@ def tag_confirm(): if not obj.exists(): abort(404) Tag.confirm_tag(tag, obj) - return redirect(obj.get_link(flask_context=True)) + + if request.referrer: + return redirect(request.referrer) + else: + return redirect(obj.get_link(flask_context=True)) @tags_ui.route('/tag/add_tags') @login_required @@ -192,22 +196,27 @@ def add_tags(): if res[1] != 200: return str(res[0]) - return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) -@tags_ui.route('/tag/delete_tag') +@tags_ui.route('/tag/delete_tag') # TODO FIX REQUEST PARAMETER @login_required @login_analyst def delete_tag(): - - object_type = request.args.get('object_type') - object_id = request.args.get('object_id') - subtype = '' # TODO: handle subtype object + object_type = request.args.get('type') + subtype = request.args.get('subtype', '') + object_id = request.args.get('id') tag = request.args.get('tag') - res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type) + res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type, subtype=subtype) if res[1] != 200: return str(res[0]) - return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) @tags_ui.route('/tag/get_all_tags') diff --git a/var/www/templates/chats_explorer/basic_card_chat.html b/var/www/templates/chats_explorer/basic_card_chat.html new file mode 100644 index 00000000..6598a472 --- /dev/null +++ b/var/www/templates/chats_explorer/basic_card_chat.html @@ -0,0 +1,67 @@ + + +
+
+

+ + + + {{ meta["svg_icon"]["icon"] }} + + + {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} : +

+
+
+ + {% if meta["tags_safe"] %} + {% if meta['icon'] %} + {{ meta['id'] }} + {% endif %} + {% else %} + + + + + {% endif %} + + + + + + + + {{meta["first_seen"]}} + + + + {{meta["last_seen"]}} + + + + + + + + + {{meta["nb_subchannels"]}}   + + + + {{meta["nb_participants"]}} + + + +
+ {% for tag in meta['tags'] %} + {{ tag }} + {% endfor %} +
+ +
+ {% include 'objects/block_object_footer_small.html' %} +
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_chat.html b/var/www/templates/chats_explorer/card_chat.html new file mode 100644 index 00000000..982297b9 --- /dev/null +++ b/var/www/templates/chats_explorer/card_chat.html @@ -0,0 +1,88 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
+
+

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :

+ {% if meta['icon'] %} +
{{ meta['id'] }}
+ {% endif %} +
    +
  • + + + + + + + + + + + + + + + + + + + + + + + +
    NameIDCreated atFirst SeenLast SeenNB Sub-ChannelsParticipants
    {{ meta['name'] }}{{ meta['id'] }}{{ meta['created_at'] }} + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['nb_subchannels'] }} + {{ meta['nb_participants']}} +
    + {% if meta['info'] %} +
  • + 
    {{ meta['info'] }}
    + {% if meta['translation_info'] %} +
    + 
    {{ meta['translation_info'] }}
    + {% endif %} +
    • + {% endif %} + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_image.html b/var/www/templates/chats_explorer/card_image.html index 058e3c0d..9e382aea 100644 --- a/var/www/templates/chats_explorer/card_image.html +++ b/var/www/templates/chats_explorer/card_image.html @@ -13,7 +13,7 @@ } -
+

{{ meta["id"] }} :

    diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index ecf45fd5..872f3b55 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -54,67 +54,9 @@
    -
    - -
    -

    {% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :

    - {% if chat['icon'] %} -
    {{ chat['id'] }}
    - {% endif %} -
      -
    • - - - - - - - - - - - - - - - - - - - - - - - -
      NameIDCreated atFirst SeenLast SeenNB Sub-ChannelsParticipants
      {{ chat['name'] }}{{ chat['id'] }}{{ chat['created_at'] }} - {% if chat['first_seen'] %} - {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} - {% endif %} - - {% if chat['last_seen'] %} - {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }} - {% endif %} - {{ chat['nb_subchannels'] }} - {{ chat['nb_participants']}} -
      - {% if chat['info'] %} -
    • - 
      {{ chat['info'] }}
      - {% if chat['translation_info'] %} -
      - 
      {{ chat['translation_info'] }}
      - {% endif %} -
      • - {% endif %} - -
    - -
    -
    - - {% for tag in chat['tags_messages'] %} - {{ tag }} {{ chat['tags_messages'][tag] }} - {% endfor %} + {% with meta=chat %} + {% include 'chats_explorer/card_chat.html' %} + {% endwith %} {% if chat['subchannels'] %}

    Sub-Channels:

    diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index a4286471..67d71a5d 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -101,6 +101,8 @@ {% if dict_object["object_type"] == "pgp" %} {% include 'correlation/metadata_card_pgp.html' %} + {% elif dict_object["object_type"] == "chat" %} + {% include 'chats_explorer/card_chat.html' %} {% elif dict_object["object_type"] == "cryptocurrency" %} {% include 'correlation/metadata_card_cryptocurrency.html' %} {% elif dict_object["object_type"] == "username" %} diff --git a/var/www/templates/modals/edit_tag.html b/var/www/templates/modals/edit_tag.html index 512c8277..ad1e4189 100644 --- a/var/www/templates/modals/edit_tag.html +++ b/var/www/templates/modals/edit_tag.html @@ -51,6 +51,6 @@ $('#edit_tags_modal').on('show.bs.modal', function (event) { tag_confirm.show(); modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid); } - modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid); + modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid); }) From d37a56b7d51bbfd80ecee656f21e9bc4789ad971 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 19 Mar 2024 11:11:56 +0100 Subject: [PATCH 12/15] chg: [chat-subchannel] add basic + default card --- bin/lib/objects/ail_objects.py | 2 +- var/www/blueprints/chats_explorer.py | 4 +- .../chats_explorer/SubChannelMessages.html | 90 +-------------- .../templates/chats_explorer/card_chat.html | 14 +++ .../chats_explorer/card_chat_subchannel.html | 103 ++++++++++++++++++ .../templates/chats_explorer/card_image.html | 2 +- .../correlation/metadata_card_chat.html | 78 ------------- .../metadata_card_cookie_name.html | 2 +- .../metadata_card_cryptocurrency.html | 2 +- .../correlation/metadata_card_cve.html | 2 +- .../correlation/metadata_card_decoded.html | 2 +- .../correlation/metadata_card_domain.html | 2 +- .../correlation/metadata_card_etag.html | 2 +- .../correlation/metadata_card_favicon.html | 2 +- .../correlation/metadata_card_hhhash.html | 2 +- .../correlation/metadata_card_item.html | 2 +- .../correlation/metadata_card_pgp.html | 2 +- .../correlation/metadata_card_title.html | 2 +- .../correlation/metadata_card_username.html | 2 +- .../correlation/show_correlation.html | 3 + .../correlation/show_relationship.html | 2 +- 21 files changed, 141 insertions(+), 181 deletions(-) create mode 100644 var/www/templates/chats_explorer/card_chat_subchannel.html delete mode 100644 var/www/templates/correlation/metadata_card_chat.html diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 14a035a4..bd4e7668 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -254,7 +254,7 @@ def get_objects_meta(objs, options=set(), flask_context=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) - meta = obj.get_meta(options={'icon', 'info', 'nb_participants'}) + meta = obj.get_meta(options={'created_at', 'icon', 'info', 'nb_messages', 'nb_participants'}) # meta['icon'] = obj.get_svg_icon() meta['svg_icon'] = obj.get_svg_icon() if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 3363a29b..5c543665 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -139,7 +139,9 @@ def objects_subchannel_messages(): else: subchannel = subchannel[0] languages = Language.get_translation_languages() - return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) + return render_template('SubChannelMessages.html', subchannel=subchannel, + ail_tags=Tag.get_modal_add_tags(subchannel['id'], subchannel['type'], subchannel['subtype']), + bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) @chats_explorer.route("/chats/explorer/thread", methods=['GET']) @login_required diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 17390a0c..e4c36d6a 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -53,93 +53,9 @@
    -
    -
    -

    {% if subchannel['chat']['name'] %}{{ subchannel['chat']['name'] }} {% else %} {{ subchannel['chat']['id'] }}{% endif %} - {% if subchannel['username'] %}{{ subchannel["username"]["id"] }} {% else %} {{ subchannel['name'] }}{% endif %} :

    {{ subchannel["id"] }} -
      -
    • -
      -
      - - - - - - - - - - - - - - - - - - - - - - - -
      NameCreated atFirst seenLast seenUsernameNb MessagesParticipants
      - {{ subchannel['name'] }} - {% if subchannel['translation_name'] %} -
      {{ subchannel['translation_name'] }}
      - {% endif %} -       		{{ subchannel["created_at"] }} - {% if subchannel['first_seen'] %} - {{ subchannel['first_seen'][0:4] }}-{{ subchannel['first_seen'][4:6] }}-{{ subchannel['first_seen'][6:8] }} - {% endif %} - - {% if subchannel['last_seen'] %} - {{ subchannel['last_seen'][0:4] }}-{{ subchannel['last_seen'][4:6] }}-{{ subchannel['last_seen'][6:8] }} - {% endif %} - - {% if 'username' in subchannel %} - {% if subchannel['username'] %} - {{ subchannel['username']['id'] }} - {% endif %} - {% endif %} - {{ subchannel['nb_messages'] }} - {{ subchannel['nb_participants']}} -
      -
      -
      -
      • - {% if subchannel['info'] %} -
    • - 
      {{ subchannel['info'] }}
      -
      • - {% endif %} -
    • -
      -
      - Tags: - {% for tag in subchannel['tags'] %} - - {% endfor %} - -
      -
      • -
    - -{# {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}#} -{# {% include 'modals/investigations_register_obj.html' %}#} -{# {% endwith %}#} -{# #} - -
    -
    - + {% with meta=subchannel %} + {% include 'chats_explorer/card_chat_subchannel.html' %} + {% endwith %} {% if subchannel['threads'] %} diff --git a/var/www/templates/chats_explorer/card_chat.html b/var/www/templates/chats_explorer/card_chat.html index 982297b9..99a0ce38 100644 --- a/var/www/templates/chats_explorer/card_chat.html +++ b/var/www/templates/chats_explorer/card_chat.html @@ -84,5 +84,19 @@ Investigations + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + \ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_chat_subchannel.html b/var/www/templates/chats_explorer/card_chat_subchannel.html new file mode 100644 index 00000000..3fd9a33e --- /dev/null +++ b/var/www/templates/chats_explorer/card_chat_subchannel.html @@ -0,0 +1,103 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
    +
    +

    {% if meta['chat']['name'] %}{{ meta['chat']['name'] }} {% else %} {{ meta['chat']['id'] }}{% endif %} - {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :

    {{ meta["id"] }} +
      +
    • +
      +
      +
    + + + + + + + + + + + + + + + + + + + + +
    NameCreated atFirst seenLast seenNb MessagesParticipants
    + {{ meta['name'] }} + {% if meta['translation_name'] %} +
    {{ meta['translation_name'] }}
    + {% endif %} +     		{{ meta["created_at"] }} + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['nb_messages'] }} + {{ meta['nb_participants']}} +
    +
    +
    + + {% if meta['info'] %} +
  • + 
    {{ meta['info'] }}
    +
    • + {% endif %} + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_image.html b/var/www/templates/chats_explorer/card_image.html index 9e382aea..e97cea81 100644 --- a/var/www/templates/chats_explorer/card_image.html +++ b/var/www/templates/chats_explorer/card_image.html @@ -33,7 +33,7 @@ - {{ meta["icon"]["icon"] }} + {{ meta["svg_icon"]["icon"] }} {{ meta['type'] }} diff --git a/var/www/templates/correlation/metadata_card_chat.html b/var/www/templates/correlation/metadata_card_chat.html deleted file mode 100644 index 4e672e1a..00000000 --- a/var/www/templates/correlation/metadata_card_chat.html +++ /dev/null @@ -1,78 +0,0 @@ - - - -{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#} -{# {% include 'modals/add_tags.html' %}#} -{#{% endwith %}#} - -{% include 'modals/edit_tag.html' %} - -
-
-

{{ dict_object["correlation_id"] }}

- {{ dict_object }} -
{{ dict_object["correlation_id"] }}
-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - -
    Object typeFirst seenLast seenNb seen
    - - - - {{ dict_object["metadata"]["icon"]["icon"] }} - - - {{ dict_object["object_type"] }} - {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    -
    -
    -
    -
    -
    -
    • - -
  • -
    -
    - Tags: - {% for tag in dict_object["metadata"]['tags'] %} - - {% endfor %} - -
    -
    • -
- - {% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %} - {% include 'modals/investigations_register_obj.html' %} - {% endwith %} - - -
-
- diff --git a/var/www/templates/correlation/metadata_card_cookie_name.html b/var/www/templates/correlation/metadata_card_cookie_name.html index 5a926182..514c8485 100644 --- a/var/www/templates/correlation/metadata_card_cookie_name.html +++ b/var/www/templates/correlation/metadata_card_cookie_name.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_cryptocurrency.html b/var/www/templates/correlation/metadata_card_cryptocurrency.html index 2698caa5..30321b67 100644 --- a/var/www/templates/correlation/metadata_card_cryptocurrency.html +++ b/var/www/templates/correlation/metadata_card_cryptocurrency.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_cve.html b/var/www/templates/correlation/metadata_card_cve.html index 1e166ddc..2a4141c9 100644 --- a/var/www/templates/correlation/metadata_card_cve.html +++ b/var/www/templates/correlation/metadata_card_cve.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index 1e292a80..25d9e499 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -32,7 +32,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata_card"]["mimetype"] }} diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html index 0e538916..ffa09537 100644 --- a/var/www/templates/correlation/metadata_card_domain.html +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -21,7 +21,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_etag.html b/var/www/templates/correlation/metadata_card_etag.html index cc599227..1ce11821 100644 --- a/var/www/templates/correlation/metadata_card_etag.html +++ b/var/www/templates/correlation/metadata_card_etag.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_favicon.html b/var/www/templates/correlation/metadata_card_favicon.html index 6b12ab3b..496cbb77 100644 --- a/var/www/templates/correlation/metadata_card_favicon.html +++ b/var/www/templates/correlation/metadata_card_favicon.html @@ -29,7 +29,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_hhhash.html b/var/www/templates/correlation/metadata_card_hhhash.html index c1474605..5b37ced3 100644 --- a/var/www/templates/correlation/metadata_card_hhhash.html +++ b/var/www/templates/correlation/metadata_card_hhhash.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_item.html b/var/www/templates/correlation/metadata_card_item.html index 194ec200..cf5395d1 100644 --- a/var/www/templates/correlation/metadata_card_item.html +++ b/var/www/templates/correlation/metadata_card_item.html @@ -18,7 +18,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_pgp.html b/var/www/templates/correlation/metadata_card_pgp.html index 03d73a2b..bb870dcd 100644 --- a/var/www/templates/correlation/metadata_card_pgp.html +++ b/var/www/templates/correlation/metadata_card_pgp.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_title.html b/var/www/templates/correlation/metadata_card_title.html index cd943349..7b1140f0 100644 --- a/var/www/templates/correlation/metadata_card_title.html +++ b/var/www/templates/correlation/metadata_card_title.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_username.html b/var/www/templates/correlation/metadata_card_username.html index e1d4dbe1..c1e45cd6 100644 --- a/var/www/templates/correlation/metadata_card_username.html +++ b/var/www/templates/correlation/metadata_card_username.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 67d71a5d..4a023967 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -99,10 +99,13 @@
+ {% set is_correlation = True %} {% if dict_object["object_type"] == "pgp" %} {% include 'correlation/metadata_card_pgp.html' %} {% elif dict_object["object_type"] == "chat" %} {% include 'chats_explorer/card_chat.html' %} + {% elif dict_object["object_type"] == "chat-subchannel" %} + {% include 'chats_explorer/card_chat_subchannel.html' %} {% elif dict_object["object_type"] == "cryptocurrency" %} {% include 'correlation/metadata_card_cryptocurrency.html' %} {% elif dict_object["object_type"] == "username" %} diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index bff41724..9ad5cc8b 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -108,7 +108,7 @@ {% elif dict_object["object_type"] == "decoded" %} {% include 'correlation/metadata_card_decoded.html' %} {% elif dict_object["object_type"] == "chat" %} - {% include 'correlation/metadata_card_chat.html' %} + {% include 'chats_explorer/card_chat.html' %} {% elif dict_object["object_type"] == "cve" %} {% include 'correlation/metadata_card_cve.html' %} {% elif dict_object["object_type"] == "domain" %} From 9a0e77dbbd059e9ced421c9ad3bc55f033f7ba19 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 19 Mar 2024 11:14:32 +0100 Subject: [PATCH 13/15] fix: [chat-subchannel] card empty chat --- bin/lib/objects/ail_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index bd4e7668..2ff619ce 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -254,7 +254,7 @@ def get_objects_meta(objs, options=set(), flask_context=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) - meta = obj.get_meta(options={'created_at', 'icon', 'info', 'nb_messages', 'nb_participants'}) + meta = obj.get_meta(options={'chat', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads'}) # meta['icon'] = obj.get_svg_icon() meta['svg_icon'] = obj.get_svg_icon() if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': From 912eadc5636821aaae19336b82418dbdeb53dcbf Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 19 Mar 2024 12:05:50 +0100 Subject: [PATCH 14/15] chg: [user-account] add default card --- bin/lib/objects/ail_objects.py | 2 +- var/www/blueprints/chats_explorer.py | 1 + .../chats_explorer/card_user_account.html | 97 +++++++++++++++++++ .../chats_explorer/user_account.html | 70 +------------ .../correlation/show_correlation.html | 2 + 5 files changed, 102 insertions(+), 70 deletions(-) create mode 100644 var/www/templates/chats_explorer/card_user_account.html diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 2ff619ce..6f4f750d 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -254,7 +254,7 @@ def get_objects_meta(objs, options=set(), flask_context=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) - meta = obj.get_meta(options={'chat', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads'}) + meta = obj.get_meta(options={'chat', 'chats', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads', 'username'}) # meta['icon'] = obj.get_svg_icon() meta['svg_icon'] = obj.get_svg_icon() if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 5c543665..49544dbb 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -270,4 +270,5 @@ def objects_user_account(): user_account = user_account[0] languages = Language.get_translation_languages() return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']), translation_languages=languages, translation_target=target) diff --git a/var/www/templates/chats_explorer/card_user_account.html b/var/www/templates/chats_explorer/card_user_account.html new file mode 100644 index 00000000..f5514967 --- /dev/null +++ b/var/www/templates/chats_explorer/card_user_account.html @@ -0,0 +1,97 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
+
+

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %}

+ {% if meta['icon'] %} +
{{ meta['id'] }}
+ {% endif %} +
    +
  • + + + + + + + + + + + + + + + + + + + +
    usernameIDFirst SeenLast SeenNB Chats
    {{ meta['username']['id'] }}{{ meta['id'] }} + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['chats'] | length }}
    + {% if meta['info'] %} +
  • + 
    {{ meta['info'] }}
    + {% if meta['translation_info'] %} +
    + 
    {{ meta['translation_info'] }}
    + {% endif %} +
    • + {% endif %} + + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html index 7d869733..308f5959 100644 --- a/var/www/templates/chats_explorer/user_account.html +++ b/var/www/templates/chats_explorer/user_account.html @@ -32,75 +32,7 @@
-
- -
-

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %}

- {% if meta['icon'] %} -
{{ meta['id'] }}
- {% endif %} -
    -
  • - - - - - - - - - - - - - - - - - - - -
    usernameIDFirst SeenLast SeenNB Chats
    {{ meta['username']['id'] }}{{ meta['id'] }} - {% if meta['first_seen'] %} - {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} - {% endif %} - - {% if meta['last_seen'] %} - {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} - {% endif %} - {{ meta['chats'] | length }}
    - {% if meta['info'] %} -
  • - 
    {{ meta['info'] }}
    - {% if meta['translation_info'] %} -
    - 
    {{ meta['translation_info'] }}
    - {% endif %} -
    • - {% endif %} - -
- -
- -{#
#} -{# {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%}#} -{# {% include 'modals/investigations_register_obj.html' %}#} -{# {% endwith %}#} -{#
#} -{# #} -{#
#} -{#
#} -
- -
-
+ {% include 'chats_explorer/card_user_account.html' %} {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} {% include 'chats_explorer/block_translation.html' %} diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 4a023967..cdd0bc64 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -106,6 +106,8 @@ {% include 'chats_explorer/card_chat.html' %} {% elif dict_object["object_type"] == "chat-subchannel" %} {% include 'chats_explorer/card_chat_subchannel.html' %} + {% elif dict_object["object_type"] == "user-account" %} + {% include 'chats_explorer/card_user_account.html' %} {% elif dict_object["object_type"] == "cryptocurrency" %} {% include 'correlation/metadata_card_cryptocurrency.html' %} {% elif dict_object["object_type"] == "username" %} From 406d72bb5256c803b97f2be7d0710bd86774eaf7 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 20 Mar 2024 11:39:41 +0100 Subject: [PATCH 15/15] chg: [requirements] add demoji --- requirements.txt | 5 ++++- update/v5.4/Update.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6fc9e511..9d965a02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ textblob>=0.15.3 html2text>=2020.1.16 beautifulsoup4>4.8.2 -#Crawler +# Crawler scrapy>2.0.0 scrapy-splash>=0.7.2 @@ -45,6 +45,9 @@ scrapy-splash>=0.7.2 gcld3 libretranslatepy +# Demoji +git+https://github.com/ail-project/demoji + #Graph numpy>1.18.1 matplotlib>3.2.1 diff --git a/update/v5.4/Update.sh b/update/v5.4/Update.sh index 25e98c30..b198f517 100755 --- a/update/v5.4/Update.sh +++ b/update/v5.4/Update.sh @@ -24,6 +24,7 @@ echo "" echo -e $GREEN"Updating python packages ..."$DEFAULT echo "" pip install -U pylacus +pip install git+https://github.com/ail-project/demoji bash ${AIL_BIN}/LAUNCH.sh -lrv