AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [lang] merge master

Browse source
This commit is contained in:
terrtia 2024-03-21 11:01:43 +01:00
commit 59ca8c5d31
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
74 changed files with 883 additions and 433 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
bin/importer/FeederImporter.py
View file

@ -103,14 +103,16 @@ class FeederImporter(AbstractImporter):
if data_obj: if data_obj:
objs.add(data_obj) objs.add(data_obj)
objs_messages = []
for obj in objs: for obj in objs:
if obj.type == 'item': # object save on disk as file (Items) if obj.type == 'item': # object save on disk as file (Items)
gzip64_content = feeder.get_gzip64_content() gzip64_content = feeder.get_gzip64_content()
return obj, f'{feeder_name} {gzip64_content}' relay_message = f'{feeder_name} {gzip64_content}'
objs_messages.append({'obj': obj, 'message': relay_message})
else: # Messages save on DB else: # Messages save on DB
if obj.exists() and obj.type != 'chat': if obj.exists() and obj.type != 'chat':
return obj, f'{feeder_name}' objs_messages.append({'obj': obj, 'message': feeder_name})
return objs_messages
class FeederModuleImporter(AbstractModule): class FeederModuleImporter(AbstractModule):
def __init__(self): def __init__(self):
@ -128,10 +130,8 @@ class FeederModuleImporter(AbstractModule):
def compute(self, message): def compute(self, message):
# TODO HANDLE Invalid JSON # TODO HANDLE Invalid JSON
json_data = json.loads(message) json_data = json.loads(message)
# TODO multiple objs + messages for obj_message in self.importer.importer(json_data):
obj, relay_message = self.importer.importer(json_data) self.add_message_to_queue(obj=obj_message['obj'], message=obj_message['message'])
####
self.add_message_to_queue(obj=obj, message=relay_message)
# Launch Importer # Launch Importer

4
bin/lib/Tag.py
View file

@ -1558,14 +1558,14 @@ def get_obj_date(object_type, object_id):
return None return None
# API QUERY # API QUERY
def api_delete_obj_tags(tags=[], object_id=None, object_type="item"): def api_delete_obj_tags(tags=[], object_id=None, object_type="item", subtype=''):
if not object_id: if not object_id:
return ({'status': 'error', 'reason': 'object id not found'}, 404) return ({'status': 'error', 'reason': 'object id not found'}, 404)
if not tags: if not tags:
return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400) return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
for tag in tags: for tag in tags:
res = delete_object_tag(tag, object_type, object_id, subtype='') res = delete_object_tag(tag, object_type, object_id, subtype=subtype)
if res: if res:
return res return res

11
bin/lib/ail_core.py
View file

@ -22,6 +22,11 @@ AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', '
AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'}
# TODO by object TYPE ????
AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded',
'domain', 'favicon', 'file-name',
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
def get_ail_uuid(): def get_ail_uuid():
ail_uuid = r_serv_db.get('ail:uuid') ail_uuid = r_serv_db.get('ail:uuid')
if not ail_uuid: if not ail_uuid:
@ -49,6 +54,9 @@ def generate_uuid():
def get_all_objects(): def get_all_objects():
return AIL_OBJECTS return AIL_OBJECTS
def is_object_type(obj_type):
return obj_type in AIL_OBJECTS
def get_objects_with_subtypes(): def get_objects_with_subtypes():
return AIL_OBJECTS_WITH_SUBTYPES return AIL_OBJECTS_WITH_SUBTYPES
@ -69,6 +77,9 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
return r_object.smembers(f'all_chat:subtypes') return r_object.smembers(f'all_chat:subtypes')
return [] return []
def get_default_correlation_objects():
return AIL_OBJECTS_CORRELATIONS_DEFAULT
def get_obj_queued(): def get_obj_queued():
return ['item', 'image'] return ['item', 'image']

4
bin/lib/ail_queues.py
View file

@ -139,6 +139,10 @@ class AILQueue:
def error(self): def error(self):
r_queues.hdel(f'modules', f'{self.pid}:{self.name}') r_queues.hdel(f'modules', f'{self.pid}:{self.name}')
def end(self):
self.clear()
r_queues.hdel(f'modules', f'{self.pid}:{self.name}')
def get_queues_modules(): def get_queues_modules():
return r_queues.hkeys('queues') return r_queues.hkeys('queues')

57
bin/lib/chats_viewer.py
View file

@ -323,6 +323,63 @@ def get_username_meta_from_global_id(username_global_id):
username = Usernames.Username(username_id, instance_uuid) username = Usernames.Username(username_id, instance_uuid)
return username.get_meta() return username.get_meta()
# TODO Filter
## Instance type
## Chats IDS
## SubChats IDS
## Threads IDS
## Daterange
def get_messages_iterator(filters={}):
for instance_uuid in get_chat_service_instances():
for chat_id in ChatServiceInstance(instance_uuid).get_chats():
chat = Chats.Chat(chat_id, instance_uuid)
# subchannels
for subchannel_gid in chat.get_subchannels():
_, _, subchannel_id = subchannel_gid.split(':', 2)
subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid)
messages, _ = subchannel._get_messages(nb=-1)
for mess in messages:
_, _, message_id = mess[0].split(':', )
yield Messages.Message(message_id)
# threads
# threads
for threads in chat.get_threads():
thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
_, _ = thread._get_messages(nb=-1)
for mess in messages:
message_id, _, message_id = mess[0].split(':', )
yield Messages.Message(message_id)
# messages
messages, _ = chat._get_messages(nb=-1)
for mess in messages:
_, _, message_id = mess[0].split(':', )
yield Messages.Message(message_id)
# threads ???
def get_nb_messages_iterator(filters={}):
nb_messages = 0
for instance_uuid in get_chat_service_instances():
for chat_id in ChatServiceInstance(instance_uuid).get_chats():
chat = Chats.Chat(chat_id, instance_uuid)
# subchannels
for subchannel_gid in chat.get_subchannels():
_, _, subchannel_id = subchannel_gid.split(':', 2)
subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid)
nb_messages += subchannel.get_nb_messages()
# threads
for threads in chat.get_threads():
thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
nb_messages += thread.get_nb_messages()
# messages
nb_messages += chat.get_nb_messages()
return nb_messages
#### API #### #### API ####
def api_get_chat_service_instance(chat_instance_uuid): def api_get_chat_service_instance(chat_instance_uuid):

2
bin/lib/objects/Chats.py
View file

@ -51,7 +51,7 @@ class Chat(AbstractChatObject):
def get_link(self, flask_context=False): def get_link(self, flask_context=False):
if flask_context: if flask_context:
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) url = url_for('chats_explorer.chats_explorer_chat', subtype=self.subtype, id=self.id)
else: else:
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
return url return url

3
bin/lib/objects/Images.py
View file

@ -71,6 +71,9 @@ class Image(AbstractDaterangeObject):
return file_content return file_content
def get_content(self, r_type='str'): def get_content(self, r_type='str'):
if r_type == 'str':
return None
else:
return self.get_file_content() return self.get_file_content()
def get_misp_object(self): def get_misp_object(self):

2
bin/lib/objects/Messages.py
View file

@ -85,6 +85,7 @@ class Message(AbstractObject):
if r_type == 'str': if r_type == 'str':
return content return content
elif r_type == 'bytes': elif r_type == 'bytes':
if content:
return content.encode() return content.encode()
def get_date(self): def get_date(self):
@ -352,7 +353,6 @@ def create(obj_id, content, translation=None, tags=[]):
message.create(content, translation=translation, tags=tags) message.create(content, translation=translation, tags=tags)
return message return message
# TODO Encode translation # TODO Encode translation

11
bin/lib/objects/UsersAccount.py
View file

@ -49,9 +49,9 @@ class UserAccount(AbstractSubtypeObject):
def get_link(self, flask_context=False): def get_link(self, flask_context=False):
if flask_context: if flask_context:
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) url = url_for('chats_explorer.objects_user_account', type=self.type, subtype=self.subtype, id=self.id)
else: else:
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' url = f'{baseurl}/objects/user-account?&subtype={self.subtype}&id={self.id}'
return url return url
def get_svg_icon(self): # TODO change icon/color def get_svg_icon(self): # TODO change icon/color
@ -127,6 +127,13 @@ class UserAccount(AbstractSubtypeObject):
def update_username_timeline(self, username_global_id, timestamp): def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id) self._get_timeline_username().add_timestamp(timestamp, username_global_id)
def get_messages(self):
messages = []
for mess in self.get_correlation('message'):
messages.append(f'message:{mess}')
return messages
def get_messages_by_chat_obj(self, chat_obj): def get_messages_by_chat_obj(self, chat_obj):
messages = [] messages = []
for mess in self.get_correlation_iter_obj(chat_obj, 'message'): for mess in self.get_correlation_iter_obj(chat_obj, 'message'):

23
bin/lib/objects/ail_objects.py
View file

@ -12,12 +12,14 @@ from lib.exceptions import AILObjectUnknown
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes, get_default_correlation_objects
from lib import correlations_engine from lib import correlations_engine
from lib import relationships_engine from lib import relationships_engine
from lib import btc_ail from lib import btc_ail
from lib import Tag from lib import Tag
from lib import chats_viewer
from lib.objects import Chats from lib.objects import Chats
from lib.objects import ChatSubChannels from lib.objects import ChatSubChannels
from lib.objects import ChatThreads from lib.objects import ChatThreads
@ -32,7 +34,7 @@ from lib.objects import FilesNames
from lib.objects import HHHashs from lib.objects import HHHashs
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects import Images from lib.objects import Images
from lib.objects.Messages import Message from lib.objects import Messages
from lib.objects import Pgps from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot from lib.objects.Screenshots import Screenshot
from lib.objects import Titles from lib.objects import Titles
@ -53,12 +55,15 @@ def is_object_subtype(obj_type):
def is_valid_object_subtype(obj_type, subtype): def is_valid_object_subtype(obj_type, subtype):
return subtype in get_object_all_subtypes(obj_type) return subtype in get_object_all_subtypes(obj_type)
def sanitize_objs_types(objs): def sanitize_objs_types(objs, default=False):
l_types = [] l_types = []
for obj in objs: for obj in objs:
if is_valid_object_type(obj): if is_valid_object_type(obj):
l_types.append(obj) l_types.append(obj)
if not l_types: if not l_types:
if default:
l_types = get_default_correlation_objects()
else:
l_types = get_all_objects() l_types = get_all_objects()
return l_types return l_types
@ -87,7 +92,7 @@ def get_object(obj_type, subtype, obj_id):
elif obj_type == 'image': elif obj_type == 'image':
return Images.Image(obj_id) return Images.Image(obj_id)
elif obj_type == 'message': elif obj_type == 'message':
return Message(obj_id) return Messages.Message(obj_id)
elif obj_type == 'screenshot': elif obj_type == 'screenshot':
return Screenshot(obj_id) return Screenshot(obj_id)
elif obj_type == 'title': elif obj_type == 'title':
@ -249,8 +254,9 @@ def get_objects_meta(objs, options=set(), flask_context=False):
def get_object_card_meta(obj_type, subtype, id, related_btc=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False):
obj = get_object(obj_type, subtype, id) obj = get_object(obj_type, subtype, id)
meta = obj.get_meta() meta = obj.get_meta(options={'chat', 'chats', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads', 'username'})
meta['icon'] = obj.get_svg_icon() # meta['icon'] = obj.get_svg_icon()
meta['svg_icon'] = obj.get_svg_icon()
if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash':
meta['sparkline'] = obj.get_sparkline() meta['sparkline'] = obj.get_sparkline()
if obj_type == 'cve': if obj_type == 'cve':
@ -293,6 +299,9 @@ def obj_iterator(obj_type, filters):
return get_all_items_objects(filters=filters) return get_all_items_objects(filters=filters)
elif obj_type == 'pgp': elif obj_type == 'pgp':
return Pgps.get_all_pgps_objects(filters=filters) return Pgps.get_all_pgps_objects(filters=filters)
elif obj_type == 'message':
return chats_viewer.get_messages_iterator(filters=filters)
def card_objs_iterators(filters): def card_objs_iterators(filters):
nb = 0 nb = 0
@ -307,6 +316,8 @@ def card_obj_iterator(obj_type, filters):
return get_nb_items_objects(filters=filters) return get_nb_items_objects(filters=filters)
elif obj_type == 'pgp': elif obj_type == 'pgp':
return Pgps.nb_all_pgps_objects(filters=filters) return Pgps.nb_all_pgps_objects(filters=filters)
elif obj_type == 'message':
return chats_viewer.get_nb_messages_iterator(filters=filters)
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
""" """

7
bin/lib/telegram.py
View file

@ -22,11 +22,8 @@ REGEX_JOIN_HASH = re.compile(r'[0-9a-zA-z-]+')
## ## ## ##
def save_item_correlation(username, item_id, item_date): def save_telegram_invite_hash(invite_hash, obj_global_id):
Username.save_item_correlation('telegram', username, item_id, item_date) r_obj.sadd('telegram:invite_code', f'{invite_hash};{obj_global_id}')
def save_telegram_invite_hash(invite_hash, item_id):
r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}')
def get_data_from_telegram_url(base_url, url_path): def get_data_from_telegram_url(base_url, url_path):
dict_url = {} dict_url = {}

6
bin/modules/ApiKey.py
View file

@ -61,7 +61,7 @@ class ApiKey(AbstractModule):
if google_api_key: if google_api_key:
print(f'found google api key: {to_print}') print(f'found google api key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{item.get_id()}') self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="google-api-key"' tag = 'infoleak:automatic-detection="google-api-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
@ -69,10 +69,10 @@ class ApiKey(AbstractModule):
# # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
if aws_access_key: if aws_access_key:
print(f'found AWS key: {to_print}') print(f'found AWS key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{item.get_id()}') self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
if aws_secret_key: if aws_secret_key:
print(f'found AWS secret key') print(f'found AWS secret key')
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}') self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="aws-key"' tag = 'infoleak:automatic-detection="aws-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

4
bin/modules/Credential.py
View file

@ -103,11 +103,11 @@ class Credential(AbstractModule):
print(message) print(message)
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{item.get_id()}' to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
# num of creds above threshold, publish an alert # num of creds above threshold, publish an alert
if nb_cred > self.criticalNumberToAlert: if nb_cred > self.criticalNumberToAlert:
print(f"========> Found more than 10 credentials in this file : {item.get_id()}") print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
self.redis_logger.warning(to_print) self.redis_logger.warning(to_print)
tag = 'infoleak:automatic-detection="credential"' tag = 'infoleak:automatic-detection="credential"'

4
bin/modules/CreditCards.py
View file

@ -86,7 +86,7 @@ class CreditCards(AbstractModule):
# print(creditcard_set) # print(creditcard_set)
to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
if creditcard_set: if creditcard_set:
mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{item.id}' mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}'
print(mess) print(mess)
self.redis_logger.warning(mess) self.redis_logger.warning(mess)
@ -96,7 +96,7 @@ class CreditCards(AbstractModule):
if r_result: if r_result:
return creditcard_set return creditcard_set
else: else:
self.redis_logger.info(f'{to_print}CreditCard related;{item.id}') self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}')
if __name__ == '__main__': if __name__ == '__main__':

2
bin/modules/Cryptocurrencies.py
View file

@ -149,7 +149,7 @@ class Cryptocurrencies(AbstractModule, ABC):
item.get_date(), item.get_date(),
item.get_basename()) item.get_basename())
self.redis_logger.warning('{}Detected {} {} private key;{}'.format( self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
to_print, len(private_keys), currency['name'], item_id)) to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
else: else:
private_keys = [] private_keys = []

2
bin/modules/CveModule.py
View file

@ -56,7 +56,7 @@ class CveModule(AbstractModule):
cve = Cves.Cve(cve_id) cve = Cves.Cve(cve_id)
cve.add(date, item) cve.add(date, item)
warning = f'{item_id} contains CVEs {cves}' warning = f'{self.obj.get_global_id()} contains CVEs {cves}'
print(warning) print(warning)
self.redis_logger.warning(warning) self.redis_logger.warning(warning)

6
bin/modules/DomClassifier.py
View file

@ -82,20 +82,20 @@ class DomClassifier(AbstractModule):
localizeddomains = self.dom_classifier.include(expression=self.cc_tld) localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
if self.cc: if self.cc:
localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
if r_result: if r_result:
return self.dom_classifier.vdomain return self.dom_classifier.vdomain
except IOError as err: except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
raise Exception(f"CRC Checksum Failed on: {item.get_id()}") raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")
if __name__ == "__main__": if __name__ == "__main__":

4
bin/modules/Duplicates.py
View file

@ -92,10 +92,10 @@ class Duplicates(AbstractModule):
Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id()) Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
if nb_duplicates: if nb_duplicates:
self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{item.get_id()}') self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}')
y = time.time() y = time.time()
print(f'{item.get_id()} Processed in {y-x} sec') print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
# self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x)) # self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x))

5
bin/modules/Global.py
View file

@ -81,10 +81,9 @@ class Global(AbstractModule):
def compute(self, message, r_result=False): # TODO move OBJ ID sanitization to importer def compute(self, message, r_result=False): # TODO move OBJ ID sanitization to importer
# Recovering the streamed message infos # Recovering the streamed message infos
gzip64encoded = message
if self.obj.type == 'item': if self.obj.type == 'item':
if gzip64encoded: if message:
# Creating the full filepath # Creating the full filepath
filename = os.path.join(self.ITEMS_FOLDER, self.obj.id) filename = os.path.join(self.ITEMS_FOLDER, self.obj.id)
@ -97,7 +96,7 @@ class Global(AbstractModule):
else: else:
# Decode compressed base64 # Decode compressed base64
decoded = base64.standard_b64decode(gzip64encoded) decoded = base64.standard_b64decode(message)
new_file_content = self.gunzip_bytes_obj(filename, decoded) new_file_content = self.gunzip_bytes_obj(filename, decoded)
# TODO REWRITE ME # TODO REWRITE ME

4
bin/modules/IPAddress.py
View file

@ -82,8 +82,8 @@ class IPAddress(AbstractModule):
matching_ips.append(address) matching_ips.append(address)
if len(matching_ips) > 0: if len(matching_ips) > 0:
self.logger.info(f'{item.get_id()} contains {len(matching_ips)} IPs') self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
self.redis_logger.warning(f'{item.get_id()} contains {item.get_id()} IPs') self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
# Tag message with IP # Tag message with IP
tag = 'infoleak:automatic-detection="ip"' tag = 'infoleak:automatic-detection="ip"'

2
bin/modules/Iban.py
View file

@ -95,7 +95,7 @@ class Iban(AbstractModule):
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1) # Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}') self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}')
# Tags # Tags
tag = 'infoleak:automatic-detection="iban"' tag = 'infoleak:automatic-detection="iban"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

24
bin/modules/Keys.py
View file

@ -63,7 +63,7 @@ class Keys(AbstractModule):
get_pgp_content = False get_pgp_content = False
if KeyEnum.PGP_MESSAGE.value in content: if KeyEnum.PGP_MESSAGE.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a PGP enc message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message')
tag = 'infoleak:automatic-detection="pgp-message"' tag = 'infoleak:automatic-detection="pgp-message"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
@ -81,21 +81,21 @@ class Keys(AbstractModule):
get_pgp_content = True get_pgp_content = True
if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message')
tag = 'infoleak:automatic-detection="pgp-private-key"' tag = 'infoleak:automatic-detection="pgp-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
get_pgp_content = True get_pgp_content = True
if KeyEnum.CERTIFICATE.value in content: if KeyEnum.CERTIFICATE.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a certificate message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message')
tag = 'infoleak:automatic-detection="certificate"' tag = 'infoleak:automatic-detection="certificate"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.RSA_PRIVATE_KEY.value in content: if KeyEnum.RSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message')
print('rsa private key message found') print('rsa private key message found')
tag = 'infoleak:automatic-detection="rsa-private-key"' tag = 'infoleak:automatic-detection="rsa-private-key"'
@ -103,7 +103,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.PRIVATE_KEY.value in content: if KeyEnum.PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message')
print('private key message found') print('private key message found')
tag = 'infoleak:automatic-detection="private-key"' tag = 'infoleak:automatic-detection="private-key"'
@ -111,7 +111,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message')
print('encrypted private key message found') print('encrypted private key message found')
tag = 'infoleak:automatic-detection="encrypted-private-key"' tag = 'infoleak:automatic-detection="encrypted-private-key"'
@ -119,7 +119,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print('openssh private key message found') print('openssh private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"' tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -127,7 +127,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message')
print('SSH2 private key message found') print('SSH2 private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"' tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -135,7 +135,7 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print('OpenVPN Static key message found') print('OpenVPN Static key message found')
tag = 'infoleak:automatic-detection="vpn-static-key"' tag = 'infoleak:automatic-detection="vpn-static-key"'
@ -143,21 +143,21 @@ class Keys(AbstractModule):
# find = True # find = True
if KeyEnum.DSA_PRIVATE_KEY.value in content: if KeyEnum.DSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message')
tag = 'infoleak:automatic-detection="dsa-private-key"' tag = 'infoleak:automatic-detection="dsa-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.EC_PRIVATE_KEY.value in content: if KeyEnum.EC_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has an ec private key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message')
tag = 'infoleak:automatic-detection="ec-private-key"' tag = 'infoleak:automatic-detection="ec-private-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
# find = True # find = True
if KeyEnum.PUBLIC_KEY.value in content: if KeyEnum.PUBLIC_KEY.value in content:
self.redis_logger.warning(f'{item.get_basename()} has a public key message') self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message')
tag = 'infoleak:automatic-detection="public-key"' tag = 'infoleak:automatic-detection="public-key"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')

2
bin/modules/LibInjection.py
View file

@ -70,7 +70,7 @@ class LibInjection(AbstractModule):
print(f"Detected (libinjection) SQL in URL: {item_id}") print(f"Detected (libinjection) SQL in URL: {item_id}")
print(unquote(url)) print(unquote(url))
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print) self.redis_logger.warning(to_print)
# Add tag # Add tag

2
bin/modules/Mail.py
View file

@ -172,7 +172,7 @@ class Mail(AbstractModule):
# for tld in mx_tlds: # for tld in mx_tlds:
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item.id}' msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
if num_valid_email > self.mail_threshold: if num_valid_email > self.mail_threshold:
print(f'{item.id} Checked {num_valid_email} e-mail(s)') print(f'{item.id} Checked {num_valid_email} e-mail(s)')
self.redis_logger.warning(msg) self.redis_logger.warning(msg)

4
bin/modules/Onion.py
View file

@ -98,8 +98,8 @@ class Onion(AbstractModule):
print(f'{domain} added to crawler queue: {task_uuid}') print(f'{domain} added to crawler queue: {task_uuid}')
else: else:
to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
# TAG Item # TAG Item
tag = 'infoleak:automatic-detection="onion"' tag = 'infoleak:automatic-detection="onion"'

2
bin/modules/Phone.py
View file

@ -62,7 +62,7 @@ class Phone(AbstractModule):
tag = 'infoleak:automatic-detection="phone-number"' tag = 'infoleak:automatic-detection="phone-number"'
self.add_message_to_queue(message=tag, queue='Tags') self.add_message_to_queue(message=tag, queue='Tags')
self.redis_logger.warning(f'{item.get_id()} contains {len(phone)} Phone numbers') self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
# # List of the regex results in the Item, may be null # # List of the regex results in the Item, may be null
# results = self.REG_PHONE.findall(content) # results = self.REG_PHONE.findall(content)

2
bin/modules/SQLInjectionDetection.py
View file

@ -53,7 +53,7 @@ class SQLInjectionDetection(AbstractModule):
print(f"Detected SQL in URL: {item.id}") print(f"Detected SQL in URL: {item.id}")
print(urllib.request.unquote(url)) print(urllib.request.unquote(url))
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item.id}' to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print) self.redis_logger.warning(to_print)
# Tag # Tag

2
bin/modules/Tags.py
View file

@ -41,7 +41,7 @@ class Tags(AbstractModule):
# Create a new tag # Create a new tag
item.add_tag(tag) item.add_tag(tag)
print(f'{item.get_id()}: Tagged {tag}') print(f'{self.obj.get_global_id()}: Tagged {tag}')
# Forward message to channel # Forward message to channel
self.add_message_to_queue(message=tag, queue='Tag_feed') self.add_message_to_queue(message=tag, queue='Tag_feed')

2
bin/modules/Telegram.py
View file

@ -62,7 +62,7 @@ class Telegram(AbstractModule):
print(f'username: {user_id}') print(f'username: {user_id}')
invite_hash = dict_url.get('invite_hash') invite_hash = dict_url.get('invite_hash')
if invite_hash: if invite_hash:
telegram.save_telegram_invite_hash(invite_hash, item.id) telegram.save_telegram_invite_hash(invite_hash, self.obj.get_global_id())
print(f'invite code: {invite_hash}') print(f'invite code: {invite_hash}')
invite_code_found = True invite_code_found = True

2
bin/modules/Urls.py
View file

@ -78,7 +78,7 @@ class Urls(AbstractModule):
except AttributeError: except AttributeError:
url = url_decoded['url'] url = url_decoded['url']
print(url, item.get_id()) print(url, self.obj.get_global_id())
self.add_message_to_queue(message=str(url), queue='Url') self.add_message_to_queue(message=str(url), queue='Url')
self.logger.debug(f"url_parsed: {url}") self.logger.debug(f"url_parsed: {url}")

3
bin/modules/abstract_module.py
View file

@ -179,7 +179,10 @@ class AbstractModule(ABC):
trace = traceback.format_tb(err.__traceback__) trace = traceback.format_tb(err.__traceback__)
trace = ''.join(trace) trace = ''.join(trace)
self.logger.critical(f"Error in module {self.module_name}: {__name__} : {err}") self.logger.critical(f"Error in module {self.module_name}: {__name__} : {err}")
if message:
self.logger.critical(f"Module {self.module_name} input message: {message}") self.logger.critical(f"Module {self.module_name} input message: {message}")
if self.obj:
self.logger.critical(f"{self.module_name} Obj: {self.obj.get_global_id()}")
self.logger.critical(trace) self.logger.critical(trace)
if isinstance(err, ModuleQueueError): if isinstance(err, ModuleQueueError):

3
bin/trackers/Retro_Hunt.py
View file

@ -88,6 +88,9 @@ class Retro_Hunt_Module(AbstractModule):
for obj in ail_objects.obj_iterator(obj_type, filters): for obj in ail_objects.obj_iterator(obj_type, filters):
self.obj = obj self.obj = obj
content = obj.get_content(r_type='bytes') content = obj.get_content(r_type='bytes')
if not content:
continue
rule.match(data=content, callback=self.yara_rules_match, rule.match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)

4
bin/trackers/Tracker_Regex.py
View file

@ -116,8 +116,8 @@ class Tracker_Regex(AbstractModule):
if ail_objects.is_filtered(obj, filters): if ail_objects.is_filtered(obj, filters):
continue continue
print(f'new tracked regex found: {tracker_name} in {obj_id}') print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}') self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

6
bin/trackers/Tracker_Term.py
View file

@ -93,7 +93,7 @@ class Tracker_Term(AbstractModule):
try: try:
dict_words_freq = Tracker.get_text_word_frequency(content) dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException: except TimeoutException:
self.redis_logger.warning(f"{obj.get_id()} processing timeout") self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout")
else: else:
signal.alarm(0) signal.alarm(0)
@ -124,8 +124,8 @@ class Tracker_Term(AbstractModule):
if ail_objects.is_filtered(obj, filters): if ail_objects.is_filtered(obj, filters):
continue continue
print(f'new tracked term {tracker_uuid} found: {tracker_name} in {obj_id}') print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}') self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(), obj_id) tracker.add(obj.get_type(), obj.get_subtype(), obj_id)

4
bin/trackers/Tracker_Typo_Squatting.py
View file

@ -75,8 +75,8 @@ class Tracker_Typo_Squatting(AbstractModule):
if ail_objects.is_filtered(obj, filters): if ail_objects.is_filtered(obj, filters):
continue continue
print(f'new tracked typosquatting found: {tracked} in {obj_id}') print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}') self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

6
bin/trackers/Tracker_Yara.py
View file

@ -62,13 +62,15 @@ class Tracker_Yara(AbstractModule):
return None return None
content = self.obj.get_content(r_type='bytes') content = self.obj.get_content(r_type='bytes')
if not content:
return None
try: try:
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60) which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match: if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}') self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}')
print(f'{self.obj.get_id()}: {yara_match}') print(f'{self.obj.get_global_id()}: {yara_match}')
except yara.TimeoutError: except yara.TimeoutError:
print(f'{self.obj.get_id()}: yara scanning timed out') print(f'{self.obj.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')

5
requirements.txt
View file

@ -37,7 +37,7 @@ textblob>=0.15.3
html2text>=2020.1.16 html2text>=2020.1.16
beautifulsoup4>4.8.2 beautifulsoup4>4.8.2
#Crawler # Crawler
scrapy>2.0.0 scrapy>2.0.0
scrapy-splash>=0.7.2 scrapy-splash>=0.7.2
@ -46,6 +46,9 @@ gcld3
libretranslatepy libretranslatepy
lexilang lexilang
# Demoji
git+https://github.com/ail-project/demoji
#Graph #Graph
numpy>1.18.1 numpy>1.18.1
matplotlib>3.2.1 matplotlib>3.2.1

46
tools/reprocess_objects.py Executable file
View file

@ -0,0 +1,46 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Reprocess AIL Objects by Object Type
================
Send ALL objects by type in queues
"""
import argparse
import os
import sys
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ail_core import is_object_type
from lib import ail_queues
from lib.objects import ail_objects
def reprocess_message_objects(object_type):
queue = ail_queues.AILQueue('FeederModuleImporter', -1)
for obj in ail_objects.obj_iterator(object_type, filters={}):
queue.send_message(obj.get_global_id(), message='reprocess')
queue.end()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Reprocess AIL Objects')
parser.add_argument('-t', '--type', type=str, help='AIL Object Type', required=True)
args = parser.parse_args()
if not args.type:
parser.print_help()
sys.exit(0)
obj_type = args.type
if not is_object_type(obj_type):
raise Exception(f'Invalid Object Type: {obj_type}')
if obj_type not in ['item', 'message']: # TODO image
raise Exception(f'Currently not supported Object Type: {obj_type}')
reprocess_message_objects(obj_type)

1
update/v5.4/Update.sh
View file

@ -25,6 +25,7 @@ echo -e $GREEN"Updating python packages ..."$DEFAULT
echo "" echo ""
pip install -U pylacus pip install -U pylacus
pip install -U lexilang pip install -U lexilang
pip install git+https://github.com/ail-project/demoji
bash ${AIL_BIN}/LAUNCH.sh -lrv bash ${AIL_BIN}/LAUNCH.sh -lrv

9
var/www/blueprints/chats_explorer.py
View file

@ -92,7 +92,9 @@ def chats_explorer_chat():
else: else:
chat = chat[0] chat = chat[0]
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label,
ail_tags=Tag.get_modal_add_tags(chat['id'], chat['type'], chat['subtype']),
translation_languages=languages, translation_target=target)
@chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET']) @chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET'])
@login_required @login_required
@ -137,7 +139,9 @@ def objects_subchannel_messages():
else: else:
subchannel = subchannel[0] subchannel = subchannel[0]
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) return render_template('SubChannelMessages.html', subchannel=subchannel,
ail_tags=Tag.get_modal_add_tags(subchannel['id'], subchannel['type'], subchannel['subtype']),
bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target)
@chats_explorer.route("/chats/explorer/thread", methods=['GET']) @chats_explorer.route("/chats/explorer/thread", methods=['GET'])
@login_required @login_required
@ -279,4 +283,5 @@ def objects_user_account():
user_account = user_account[0] user_account = user_account[0]
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label, return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
translation_languages=languages, translation_target=target) translation_languages=languages, translation_target=target)

12
var/www/blueprints/correlation.py
View file

@ -165,7 +165,7 @@ def show_correlation():
related_btc = bool(request.args.get('related_btc', False)) related_btc = bool(request.args.get('related_btc', False))
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
# check if obj_id exist # check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id): if not ail_objects.exists_obj(obj_type, subtype, obj_id):
@ -190,8 +190,11 @@ def show_correlation():
else: else:
dict_object["subtype"] = '' dict_object["subtype"] = ''
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
dict_object["metadata_card"]['tags_safe'] = True
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data()) tags_selector_data=Tag.get_tags_selector_data(),
meta=dict_object["metadata_card"],
ail_tags=dict_object["metadata_card"]["add_tags_modal"])
@correlation.route('/correlation/get/description') @correlation.route('/correlation/get/description')
@login_required @login_required
@ -206,7 +209,10 @@ def get_description():
return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
# object exist # object exist
else: else:
res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'}, options = {'icon', 'tags', 'tags_safe'}
if obj_type == 'message':
options.add('content')
res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options=options,
flask_context=True) flask_context=True)
if 'tags' in res: if 'tags' in res:
res['tags'] = list(res['tags']) res['tags'] = list(res['tags'])

8
var/www/blueprints/investigations_b.py
View file

@ -209,6 +209,14 @@ def unregister_investigation():
def get_investigations_selector_json(): def get_investigations_selector_json():
return jsonify(Investigations.get_investigations_selector()) return jsonify(Investigations.get_investigations_selector())
@investigations_b.route("/object/gid")
@login_required
@login_read_only
def get_object_gid():
obj_global_id = request.args.get('gid')
ail_obj = ail_objects.get_obj_from_global_id(obj_global_id)
url = ail_obj.get_link(flask_context=True)
return redirect(url)
# #
# @investigations_b.route("/object/item") #completely shows the paste in a new tab # @investigations_b.route("/object/item") #completely shows the paste in a new tab

21
var/www/blueprints/tags_ui.py
View file

@ -170,6 +170,10 @@ def tag_confirm():
if not obj.exists(): if not obj.exists():
abort(404) abort(404)
Tag.confirm_tag(tag, obj) Tag.confirm_tag(tag, obj)
if request.referrer:
return redirect(request.referrer)
else:
return redirect(obj.get_link(flask_context=True)) return redirect(obj.get_link(flask_context=True))
@tags_ui.route('/tag/add_tags') @tags_ui.route('/tag/add_tags')
@ -192,21 +196,26 @@ def add_tags():
if res[1] != 200: if res[1] != 200:
return str(res[0]) return str(res[0])
if request.referrer:
return redirect(request.referrer)
else:
return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True))
@tags_ui.route('/tag/delete_tag') @tags_ui.route('/tag/delete_tag') # TODO FIX REQUEST PARAMETER
@login_required @login_required
@login_analyst @login_analyst
def delete_tag(): def delete_tag():
object_type = request.args.get('type')
object_type = request.args.get('object_type') subtype = request.args.get('subtype', '')
object_id = request.args.get('object_id') object_id = request.args.get('id')
subtype = '' # TODO: handle subtype object
tag = request.args.get('tag') tag = request.args.get('tag')
res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type) res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type, subtype=subtype)
if res[1] != 200: if res[1] != 200:
return str(res[0]) return str(res[0])
if request.referrer:
return redirect(request.referrer)
else:
return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True))

3
var/www/modules/dashboard/Flask_dashboard.py
View file

@ -44,6 +44,7 @@ def event_stream():
pubsub.psubscribe("Script" + '.*') pubsub.psubscribe("Script" + '.*')
for msg in pubsub.listen(): for msg in pubsub.listen():
# print(msg)
type = msg['type'] type = msg['type']
pattern = msg['pattern'] pattern = msg['pattern']
channel = msg['channel'] channel = msg['channel']
@ -77,7 +78,7 @@ def dashboard_alert(log):
log = log[46:].split(';') log = log[46:].split(';')
if len(log) == 6: if len(log) == 6:
date_time = datetime_from_utc_to_local(utc_str) date_time = datetime_from_utc_to_local(utc_str)
path = url_for('objects_item.showItem', id=log[5]) path = url_for('investigations_b.get_object_gid', gid=log[5])
res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
'paste': log[3], 'message': log[4], 'path': path} 'paste': log[3], 'message': log[4], 'path': path}

4
var/www/modules/dashboard/templates/index.html
View file

@ -162,7 +162,7 @@
</body> </body>
<script> var url_showSavedPath = "{{ url_for('objects_item.showItem') }}"; </script> <script> var url_showSavedPath = "{{ url_for('investigations_b.get_object_gid') }}"; </script>
<script> <script>
$("#page-Dashboard").addClass("active"); $("#page-Dashboard").addClass("active");
@ -181,6 +181,8 @@
var msage = document.createElement('TD') var msage = document.createElement('TD')
var inspect = document.createElement('TD') var inspect = document.createElement('TD')
// console.log(d)
tr.className = "table-log-warning"; tr.className = "table-log-warning";
time.appendChild(document.createTextNode(d.time)) time.appendChild(document.createTextNode(d.time))
chan.appendChild(document.createTextNode('Script')) chan.appendChild(document.createTextNode('Script'))

26
var/www/static/js/indexjavascript.js
View file

@ -212,18 +212,18 @@ function create_log_table(obj_json) {
tr.className = "table-danger" tr.className = "table-danger"
} }
source_link = document.createElement("A"); // source_link = document.createElement("A");
if (parsedmess[1] == "slexy.org"){ // if (parsedmess[1] == "slexy.org"){
source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0]; // source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0];
} // }
else{ // else{
source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0]; // source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
} // }
source_link.setAttribute("HREF",source_url); // source_link.setAttribute("HREF",source_url);
source_link.setAttribute("TARGET", "_blank"); // src.appendChild(source_link);
source_link.appendChild(document.createTextNode(parsedmess[1]));
src.appendChild(document.createTextNode(parsedmess[1]));
src.appendChild(source_link);
var now = new Date(); var now = new Date();
var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds()); var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds());
@ -250,8 +250,10 @@ function create_log_table(obj_json) {
msage.appendChild(document.createTextNode(message.join(" "))); msage.appendChild(document.createTextNode(message.join(" ")));
// console.log(parsedmess)
var paste_path = parsedmess[5]; var paste_path = parsedmess[5];
var url_to_saved_paste = url_showSavedPath+"?id="+paste_path; var url_to_saved_paste = url_showSavedPath+"?gid="+paste_path;
var action_icon_a = document.createElement("A"); var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank"); action_icon_a.setAttribute("TARGET", "_blank");

90
var/www/templates/chats_explorer/SubChannelMessages.html
View file

@ -53,93 +53,9 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="card my-3"> {% with meta=subchannel %}
<div class="card-header" style="background-color:#d9edf7;font-size: 15px"> {% include 'chats_explorer/card_chat_subchannel.html' %}
<h3 class="text-secondary">{% if subchannel['chat']['name'] %}{{ subchannel['chat']['name'] }} {% else %} {{ subchannel['chat']['id'] }}{% endif %} - {% if subchannel['username'] %}{{ subchannel["username"]["id"] }} {% else %} {{ subchannel['name'] }}{% endif %} :</h3> {{ subchannel["id"] }} {% endwith %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Name</th>
<th>Created at</th>
<th>First seen</th>
<th>Last seen</th>
<th>Username</th>
<th>Nb Messages</th>
<th>Participants</th>
</tr>
</thead>
<tbody>
<tr>
<td>
{{ subchannel['name'] }}
{% if subchannel['translation_name'] %}
<div class="text-secondary">{{ subchannel['translation_name'] }}</div>
{% endif %}
</td>
<td>{{ subchannel["created_at"] }}</td>
<td>
{% if subchannel['first_seen'] %}
{{ subchannel['first_seen'][0:4] }}-{{ subchannel['first_seen'][4:6] }}-{{ subchannel['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if subchannel['last_seen'] %}
{{ subchannel['last_seen'][0:4] }}-{{ subchannel['last_seen'][4:6] }}-{{ subchannel['last_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if 'username' in subchannel %}
{% if subchannel['username'] %}
{{ subchannel['username']['id'] }}
{% endif %}
{% endif %}
</td>
<td>{{ subchannel['nb_messages'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat-subchannel&subtype={{ subchannel['subtype'] }}&id={{ subchannel['id'] }}"><i class="far fa-user-circle"></i> {{ subchannel['nb_participants']}}</a>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</li>
{% if subchannel['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ subchannel['info'] }}</pre>
</li>
{% endif %}
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in subchannel['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ subchannel["subtype"] }}" data-objid="{{ subchannel["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{# {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}#}
{# {% include 'modals/investigations_register_obj.html' %}#}
{# {% endwith %}#}
{# <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
{# <i class="fas fa-microscope"></i> Investigations#}
{# </button>#}
</div>
</div>
{% if subchannel['threads'] %} {% if subchannel['threads'] %}
<table id="tablethreads" class="table"> <table id="tablethreads" class="table">

67
var/www/templates/chats_explorer/basic_card_chat.html Normal file
View file

@ -0,0 +1,67 @@
<style>
.object_image {
filter: blur(5px);
}
</style>
<div class="card">
<div class="card-header">
<h4 class="text-secondary mb-0">
<svg height="30" width="30">
<g class="nodes">
<circle cx="15" cy="15" r="15" fill="{{ meta["svg_icon"]["color"] }}"></circle>
<text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["svg_icon"]["style"] }}" font-size="16px">{{ meta["svg_icon"]["icon"] }}</text>
</g>
</svg>
{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :
</h4>
</div>
<div class="card-body py-0">
<span class="">
{% if meta["tags_safe"] %}
{% if meta['icon'] %}
<span><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="my-1" alt="{{ meta['id'] }}" width="200" height="200"></span>
{% endif %}
{% else %}
<span class="my-2 fa-stack fa-8x">
<i class="fas fa-stack-1x fa-image"></i>
<i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
</span>
{% endif %}
</span>
<span>
<span class="badge badge-dark">
<span class="badge badge-info" style="font-size: 0.8rem;">
<i class="fas fa-hourglass-start"></i>
</span>
{{meta["first_seen"]}}
<span class="badge badge-light mx-1" style="font-size: 1rem;">
<i class="far fa-calendar-alt"></i>
</span>
{{meta["last_seen"]}}
<span class="badge badge-secondary" style="font-size: 0.8rem;">
<i class="fas fa-hourglass-end"></i>
</span>
</span>
<span class="badge badge-dark">
<span class="badge badge-info" style="font-size: 0.8rem;">
<i class="far fa-comments"></i>
</span>
{{meta["nb_subchannels"]}}&nbsp;&nbsp;
<span class="badge badge-info" style="font-size: 0.8rem;">
<i class="fas fa-user-circle"></i>
</span>
{{meta["nb_participants"]}}
</span>
</span>
<div class="">
{% for tag in meta['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</div>
</div>
{% include 'objects/block_object_footer_small.html' %}
</div>

57
var/www/templates/chats_explorer/basic_card_image.html Normal file
View file

@ -0,0 +1,57 @@
<style>
.object_image {
filter: blur(5px);
}
</style>
<div class="card">
<div class="card-header">
<h4 class="text-secondary">
<svg height="30" width="30">
<g class="nodes">
<circle cx="15" cy="15" r="15" fill="{{ meta["icon"]["color"] }}"></circle>
<text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["icon"]["style"] }}" font-size="16px">{{ meta["icon"]["icon"] }}</text>
</g>
</svg>
{{ meta["id"] }} :
</h4>
</div>
<div class="card-body pt-0">
<div class="text-center">
{% if meta["tags_safe"] %}
<img class="object_image mb-1" src="{{ url_for('objects_image.image', filename=meta['id'])}}" style="max-height: 500px">
{% else %}
<span class="my-2 fa-stack fa-8x">
<i class="fas fa-stack-1x fa-image"></i>
<i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
</span>
{% endif %}
</div>
<div>
<span class="badge badge-dark">
<span data-toggle="tooltip" data-placement="top" title="Tooltip on top">
<span class="badge badge-info" style="font-size: 0.8rem;">
<i class="fas fa-hourglass-start"></i>
</span>
{{meta["first_seen"]}}
</span>
<span class="badge badge-light mx-1" style="font-size: 1rem;">
<i class="far fa-calendar-alt"></i>
</span>
{{meta["last_seen"]}}
<span class="badge badge-secondary" style="font-size: 0.8rem;">
<i class="fas fa-hourglass-end"></i>
</span>
</span>
</div>
<div class="">
{% for tag in meta['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</div>
</div>
{% include 'objects/block_object_footer_small.html' %}
</div>

102
var/www/templates/chats_explorer/card_chat.html Normal file
View file

@ -0,0 +1,102 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{% with modal_add_tags=ail_tags %}
{% include 'modals/add_tags.html' %}
{% endwith %}
{% include 'modals/edit_tag.html' %}
<div class="card my-1">
<div class="card-header">
<h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :</h4>
{% if meta['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="mb-2" alt="{{ meta['id'] }}" width="200" height="200"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th>Name</th>
<th>ID</th>
<th>Created at</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Sub-Channels</th>
<th>Participants</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{ meta['name'] }}</td>
<td>{{ meta['id'] }}</td>
<td>{{ meta['created_at'] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['nb_subchannels'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}"><i class="far fa-user-circle"></i> {{ meta['nb_participants']}}</a>
</td>
</tr>
</tbody>
</table>
{% if meta['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ meta['info'] }}</pre>
{% if meta['translation_info'] %}
<hr class="m-1">
<pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
<li class="list-group-item py-0">
<div class="my-2">
Tags:
{% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}"
data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="{{ meta['type'] }}" data-objsubtype="{{ meta['subtype'] }}" data-objid="{{ meta["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
<span class="mb-2 float-right">
{% if is_correlation %}
<a href="{{ url_for('chats_explorer.chats_explorer_chat')}}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="fas fa-comments"></i> Show Object</button>
</a>
{% else %}
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>
{% endif %}
</span>
</div>
</div>

103
var/www/templates/chats_explorer/card_chat_subchannel.html Normal file
View file

@ -0,0 +1,103 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{% with modal_add_tags=ail_tags %}
{% include 'modals/add_tags.html' %}
{% endwith %}
{% include 'modals/edit_tag.html' %}
<div class="card my-1">
<div class="card-header">
<h3 class="text-secondary">{% if meta['chat']['name'] %}{{ meta['chat']['name'] }} {% else %} {{ meta['chat']['id'] }}{% endif %} - {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :</h3> {{ meta["id"] }}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Name</th>
<th>Created at</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb Messages</th>
<th>Participants</th>
</tr>
</thead>
<tbody>
<tr>
<td>
{{ meta['name'] }}
{% if meta['translation_name'] %}
<div class="text-secondary">{{ meta['translation_name'] }}</div>
{% endif %}
</td>
<td>{{ meta["created_at"] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['nb_messages'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat-subchannel&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}"><i class="far fa-user-circle"></i> {{ meta['nb_participants']}}</a>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</li>
{% if meta['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ meta['info'] }}</pre>
</li>
{% endif %}
<li class="list-group-item py-0">
<div class="my-2">
Tags:
{% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}"
data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="{{ meta['type'] }}" data-objsubtype="{{ meta['subtype'] }}" data-objid="{{ meta["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
<span class="mb-2 float-right">
{% if is_correlation %}
<a href="{{ url_for('chats_explorer.objects_subchannel_messages')}}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="far fa-comments"></i> Show Object</button>
</a>
{% else %}
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>
{% endif %}
</span>
</div>
</div>

104
var/www/templates/chats_explorer/card_image.html Normal file
View file

@ -0,0 +1,104 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{% with modal_add_tags=ail_tags %}
{% include 'modals/add_tags.html' %}
{% endwith %}
{% include 'modals/edit_tag.html' %}
<style>
.object_image {
filter: blur(5px);
}
</style>
<div class="card my-1">
<div class="card-header">
<h4 class="text-secondary">{{ meta["id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th></th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ meta["svg_icon"]["style"] }}" font-size="16px">{{ meta["svg_icon"]["icon"] }}</text>
</g>
</svg>
{{ meta['type'] }}
</td>
<td>
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
</td>
<td>
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
</td>
<td>{{ meta['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</li>
<li class="list-group-item py-0">
<div id="accordion_image" class="my-3">
<div class="card">
<div class="card-header py-1" id="headingImage">
<button class="btn w-100 collapsed rotate" data-toggle="collapse" data-target="#collapseImage" aria-expanded="false" aria-controls="collapseImage">
<span class="row text-left">
<div class="col-11">
<span class="mt-2">
<i class="far fa-image"></i> Show Image&nbsp;&nbsp;
</span>
</div>
<div class="col-1 text-primary">
<i class="fas fa-chevron-circle-down"></i>
</div>
</span>
</button>
</div>
<div id="collapseImage" class="collapse show" aria-labelledby="headingImage" data-parent="#accordion_image">
<div class="card-body text-center">
{% include 'objects/image/block_blur_img_slider.html' %}
<img class="object_image mb-1" src="{{ url_for('objects_image.image', filename=meta['id'])}}">
</div>
</div>
</div>
</div>
</li>
<li class="list-group-item py-0">
<div class="my-2">
Tags:
{% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}"
data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="{{ meta['type'] }}" data-objsubtype="" data-objid="{{ meta["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='image', obj_id=meta['id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>

97
var/www/templates/chats_explorer/card_user_account.html Normal file
View file

@ -0,0 +1,97 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{% with modal_add_tags=ail_tags %}
{% include 'modals/add_tags.html' %}
{% endwith %}
{% include 'modals/edit_tag.html' %}
<div class="card my-1">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %} </h4>
{% if meta['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="mb-2" alt="{{ meta['id'] }}" width="250" height="250"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th>username</th>
<th>ID</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Chats</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{ meta['username']['id'] }}</td>
<td>{{ meta['id'] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['chats'] | length }}</td>
</tr>
</tbody>
</table>
{% if meta['info'] %}
<li class="list-group-item py-1">
<pre class="my-0">{{ meta['info'] }}</pre>
{% if meta['translation_info'] %}
<hr>
<pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
</li>
<li class="list-group-item py-0">
<div class="my-2">
Tags:
{% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}"
data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="{{ meta['type'] }}" data-objsubtype="{{ meta['subtype'] }}" data-objid="{{ meta["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
<span class="mb-2 float-right">
{% if is_correlation %}
<a href="{{ url_for('chats_explorer.objects_user_account')}}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="fas fa-user-circle"></i> Show Object</button>
</a>
{% else %}
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
<button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
{# <span class="badge badge-warning">{{ meta['nb_correlations'] }}</span>#}
</button>
</a>
{% endif %}
</span>
</div>
</div>

64
var/www/templates/chats_explorer/chat_viewer.html
View file

@ -54,67 +54,9 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="card my-3"> {% with meta=chat %}
{% include 'chats_explorer/card_chat.html' %}
<div class="card-header" style="background-color:#d9edf7;font-size: 15px"> {% endwith %}
<h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>
{% if chat['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th>Name</th>
<th>ID</th>
<th>Created at</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Sub-Channels</th>
<th>Participants</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{ chat['name'] }}</td>
<td>{{ chat['id'] }}</td>
<td>{{ chat['created_at'] }}</td>
<td>
{% if chat['first_seen'] %}
{{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if chat['last_seen'] %}
{{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ chat['nb_subchannels'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat&subtype={{ chat['subtype'] }}&id={{ chat['id'] }}"><i class="far fa-user-circle"></i> {{ chat['nb_participants']}}</a>
</td>
</tr>
</tbody>
</table>
{% if chat['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ chat['info'] }}</pre>
{% if chat['translation_info'] %}
<hr class="m-1">
<pre class="my-0 text-secondary">{{ chat['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
</li>
</ul>
</div>
</div>
{% for tag in chat['tags_messages'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ chat['tags_messages'][tag] }}</span></span>
{% endfor %}
{% if chat['subchannels'] %} {% if chat['subchannels'] %}
<h4 class="mt-2">Sub-Channels:</h4> <h4 class="mt-2">Sub-Channels:</h4>

70
var/www/templates/chats_explorer/user_account.html
View file

@ -32,75 +32,7 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="card my-3"> {% include 'chats_explorer/card_user_account.html' %}
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %} </h4>
{% if meta['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="mb-2" alt="{{ meta['id'] }}" width="250" height="250"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th>username</th>
<th>ID</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Chats</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{ meta['username']['id'] }}</td>
<td>{{ meta['id'] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['chats'] | length }}</td>
</tr>
</tbody>
</table>
{% if meta['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ meta['info'] }}</pre>
{% if meta['translation_info'] %}
<hr>
<pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
</li>
</ul>
<div class="d-flex flex-row-reverse bd-highlight">
<div>
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}" target="_blank">
<button class="btn btn-lg btn-info"><i class="fas fa-project-diagram"></i> Correlations Graph</button>
</a>
</div>
{# <div>#}
{# {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%}#}
{# {% include 'modals/investigations_register_obj.html' %}#}
{# {% endwith %}#}
{# <div class="mr-2">#}
{# <button type="button" class="btn btn-lg btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
{# <i class="fas fa-microscope"></i> Investigations#}
{# </button>#}
{# </div>#}
{# </div>#}
</div>
</div>
</div>
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %} {% include 'chats_explorer/block_translation.html' %}

78
var/www/templates/correlation/metadata_card_chat.html
View file

@ -1,78 +0,0 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#}
{# {% include 'modals/add_tags.html' %}#}
{#{% endwith %}#}
{% include 'modals/edit_tag.html' %}
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4>{{ dict_object["correlation_id"] }}</h4>
{{ dict_object }}
<div class="text-secondary">{{ dict_object["correlation_id"] }}</div>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata"]["icon"]["icon"] }}</text>
</g>
</svg>
{{ dict_object["object_type"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in dict_object["metadata"]['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="cookie-name" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>

2
var/www/templates/correlation/metadata_card_cookie_name.html
View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["object_type"] }} {{ dict_object["object_type"] }}

2
var/www/templates/correlation/metadata_card_cryptocurrency.html
View file

@ -31,7 +31,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

2
var/www/templates/correlation/metadata_card_cve.html
View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
</td> </td>

2
var/www/templates/correlation/metadata_card_decoded.html
View file

@ -32,7 +32,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata_card"]["mimetype"] }} {{ dict_object["metadata_card"]["mimetype"] }}

2
var/www/templates/correlation/metadata_card_domain.html
View file

@ -21,7 +21,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

2
var/www/templates/correlation/metadata_card_etag.html
View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["object_type"] }} {{ dict_object["object_type"] }}

2
var/www/templates/correlation/metadata_card_favicon.html
View file

@ -29,7 +29,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["object_type"] }} {{ dict_object["object_type"] }}

2
var/www/templates/correlation/metadata_card_hhhash.html
View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["object_type"] }} {{ dict_object["object_type"] }}

2
var/www/templates/correlation/metadata_card_item.html
View file

@ -18,7 +18,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

2
var/www/templates/correlation/metadata_card_pgp.html
View file

@ -31,7 +31,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

2
var/www/templates/correlation/metadata_card_title.html
View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["object_type"] }} {{ dict_object["object_type"] }}

2
var/www/templates/correlation/metadata_card_username.html
View file

@ -31,7 +31,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ dict_object["metadata_card"]["svg_icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["svg_icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

42
var/www/templates/correlation/show_correlation.html
View file

@ -99,8 +99,15 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
{% set is_correlation = True %}
{% if dict_object["object_type"] == "pgp" %} {% if dict_object["object_type"] == "pgp" %}
{% include 'correlation/metadata_card_pgp.html' %} {% include 'correlation/metadata_card_pgp.html' %}
{% elif dict_object["object_type"] == "chat" %}
{% include 'chats_explorer/card_chat.html' %}
{% elif dict_object["object_type"] == "chat-subchannel" %}
{% include 'chats_explorer/card_chat_subchannel.html' %}
{% elif dict_object["object_type"] == "user-account" %}
{% include 'chats_explorer/card_user_account.html' %}
{% elif dict_object["object_type"] == "cryptocurrency" %} {% elif dict_object["object_type"] == "cryptocurrency" %}
{% include 'correlation/metadata_card_cryptocurrency.html' %} {% include 'correlation/metadata_card_cryptocurrency.html' %}
{% elif dict_object["object_type"] == "username" %} {% elif dict_object["object_type"] == "username" %}
@ -121,6 +128,8 @@
{% include 'correlation/metadata_card_etag.html' %} {% include 'correlation/metadata_card_etag.html' %}
{% elif dict_object["object_type"] == "hhhash" %} {% elif dict_object["object_type"] == "hhhash" %}
{% include 'correlation/metadata_card_hhhash.html' %} {% include 'correlation/metadata_card_hhhash.html' %}
{% elif dict_object["object_type"] == "image" %}
{% include 'chats_explorer/card_image.html' %}
{% elif dict_object["object_type"] == "item" %} {% elif dict_object["object_type"] == "item" %}
{% include 'correlation/metadata_card_item.html' %} {% include 'correlation/metadata_card_item.html' %}
{% elif dict_object["object_type"] == "favicon" %} {% elif dict_object["object_type"] == "favicon" %}
@ -204,6 +213,22 @@
<div class="card mb-3"> <div class="card mb-3">
<div class="card-body text-center px-0 py-0"> <div class="card-body text-center px-0 py-0">
<ul class="list-group">
<li class="list-group-item list-group-item-secondary"><i class="fas fa-project-diagram"></i> Direct Correlations</li>
<li class="list-group-item text-left">
{% for obj_type in dict_object['nb_correl'] %}
<div class="row">
<div class="col-8">
{{ obj_type }}
</div>
<div class="col-4">
<span class="badge badge-primary">{{ dict_object['nb_correl'][obj_type] }}</span>
</div>
</div>
{% endfor %}
</li>
</ul>
<ul class="list-group"> <ul class="list-group">
<li class="list-group-item list-group-item-info">Select Correlation</li> <li class="list-group-item list-group-item-info">Select Correlation</li>
<form action="{{ url_for('correlation.show_correlation') }}" method="post"> <form action="{{ url_for('correlation.show_correlation') }}" method="post">
@ -343,21 +368,6 @@
</p> </p>
</li> </li>
</ul> </ul>
<ul class="list-group">
<li class="list-group-item list-group-item-secondary"><i class="fas fa-project-diagram"></i> Direct Correlations</li>
<li class="list-group-item text-left">
{% for obj_type in dict_object['nb_correl'] %}
<div class="row">
<div class="col-8">
{{ obj_type }}
</div>
<div class="col-4">
<span class="badge badge-primary">{{ dict_object['nb_correl'][obj_type] }}</span>
</div>
</div>
{% endfor %}
</li>
</ul>
</div> </div>
</div> </div>
@ -714,7 +724,9 @@ if (d.popover) {
} }
desc = desc + "</div></dd>" desc = desc + "</div></dd>"
} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") { } else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
if (data[key]) {
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>" desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
}
} }
}); });
desc = desc + "</dl>" desc = desc + "</dl>"

2
var/www/templates/correlation/show_relationship.html
View file

@ -108,7 +108,7 @@
{% elif dict_object["object_type"] == "decoded" %} {% elif dict_object["object_type"] == "decoded" %}
{% include 'correlation/metadata_card_decoded.html' %} {% include 'correlation/metadata_card_decoded.html' %}
{% elif dict_object["object_type"] == "chat" %} {% elif dict_object["object_type"] == "chat" %}
{% include 'correlation/metadata_card_chat.html' %} {% include 'chats_explorer/card_chat.html' %}
{% elif dict_object["object_type"] == "cve" %} {% elif dict_object["object_type"] == "cve" %}
{% include 'correlation/metadata_card_cve.html' %} {% include 'correlation/metadata_card_cve.html' %}
{% elif dict_object["object_type"] == "domain" %} {% elif dict_object["object_type"] == "domain" %}

2
var/www/templates/domains/card_img_domain.html
View file

@ -55,7 +55,7 @@
<span class="badge badge-light mx-1" style="font-size: 1rem;"> <span class="badge badge-light mx-1" style="font-size: 1rem;">
<i class="far fa-calendar-alt"></i> <i class="far fa-calendar-alt"></i>
</span> </span>
{{dict_domain["first_seen"]}} {{dict_domain["last_seen"]}}
<span class="badge badge-secondary" style="font-size: 0.8rem;"> <span class="badge badge-secondary" style="font-size: 0.8rem;">
<i class="fas fa-hourglass-end"></i> <i class="fas fa-hourglass-end"></i>
</span> </span>

4
var/www/templates/hunter/add_retro_hunt_task.html
View file

@ -78,6 +78,10 @@
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj"> <input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj">
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label> <label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
</div> </div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
<label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i>&nbsp;Messages</label>
</div>
{# <div class="custom-control custom-switch mt-1">#} {# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#} {# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#} {# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}

2
var/www/templates/modals/edit_tag.html
View file

@ -51,6 +51,6 @@ $('#edit_tags_modal').on('show.bs.modal', function (event) {
tag_confirm.show(); tag_confirm.show();
modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid); modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid);
} }
modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid); modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid);
}) })
</script> </script>

12
var/www/templates/objects/block_object_footer_small.html Normal file
View file

@ -0,0 +1,12 @@
<div class="d-flex flex-row-reverse bd-highlight">
{# <div>#}
{# <a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{ meta['type'] }}{% if meta['subtype'] %}&subtype={{ meta['subtype'] }}{%endif%}&id={{ meta['id'] }}&lvl=0">#}
{# <img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">#}
{# </a>#}
{# </div>#}
<div>
<a class="btn btn-outline-light" href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}{% if meta['subtype'] %}&subtype={{ meta['subtype'] }}{%endif%}&id={{ meta['id'] }}" target="_blank" style="font-size: 15px">
<i class="fas fa-project-diagram text-secondary"></i>
</a>
</div>
</div>